mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
Merge branch 'main' into fix/clickhouse-idle-before-backup
main now names the ClickHouse archive in the controller and restores the archive a Backup recorded, in the same strategy script this branch gates on an idle sidecar. The merged script keeps both: the archive name and its restore selection from main, the idle gate, retried polls and lost-operation check from this branch. A retried backup is taken over through the same idle gate as a restore, matched on the whole command line the sidecar logs. With resume requested that line reads "create_remote --resume <name>", so the gate matches that form; matching on the name alone would also take another operation's row ending in the same name for this one. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
commit
89f8cc02ee
29 files changed
+3478
-94
No files matched your search
@@ -37,7 +37,14 @@ type BackupArtifact struct {
|
||||
// For example: s3://bucket/prefix/file.tar.gz
|
||||
URI string `json:"uri"`
|
||||
|
||||
// SizeBytes is the size of the artifact in bytes, if known.
|
||||
// SizeBytes is the stored size in bytes of the files this backup owns,
|
||||
// if known: the dump of a MariaDB backup, for example, or the base backup
|
||||
// of a CNPG one, which leaves out the WAL archive the server's backups
|
||||
// share and a point-in-time restore also reads. Strategies that cannot
|
||||
// attribute stored bytes to a single backup, such as Velero's, whose
|
||||
// repository deduplicates across backups, leave it unset. Drivers never
|
||||
// write 0 for a size they cannot measure, so an absent value means
|
||||
// "unknown", not "empty".
|
||||
// +optional
|
||||
SizeBytes int64 `json:"sizeBytes,omitempty"`
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ require (
|
||||
github.com/go-logr/logr v1.4.4
|
||||
github.com/go-logr/zapr v1.3.0
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0
|
||||
github.com/minio/minio-go/v7 v7.0.98
|
||||
github.com/onsi/ginkgo/v2 v2.33.0
|
||||
github.com/onsi/gomega v1.43.1
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
@@ -63,6 +64,7 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.4 // indirect
|
||||
github.com/gkampitakis/go-snaps v0.5.23 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v1.0.1 // indirect
|
||||
github.com/go-openapi/jsonreference v1.0.2 // indirect
|
||||
@@ -93,18 +95,25 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/moby/spdystream v0.5.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_model v0.6.3 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/procfs v0.22.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.16.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
github.com/tidwall/match v1.2.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.7.0 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect
|
||||
|
||||
@@ -61,6 +61,8 @@ github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z
|
||||
github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
|
||||
github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg=
|
||||
github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -141,6 +143,11 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -151,6 +158,12 @@ github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg=
|
||||
github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
|
||||
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
|
||||
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
|
||||
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
|
||||
github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y=
|
||||
github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -165,6 +178,8 @@ github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8
|
||||
github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||
github.com/onsi/gomega v1.43.1 h1:vGIPFuYrIO6/0Z09s0I0QQQgFchiX4+tb1re3MScJYo=
|
||||
github.com/onsi/gomega v1.43.1/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
@@ -180,6 +195,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
|
||||
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
|
||||
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
@@ -206,6 +223,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
|
||||
github.com/vmware-tanzu/velero v1.18.3 h1:9uwN5yxTWQl9PsF5ljX4xiG+7VIq3I9mxOQ0DyyZ31A=
|
||||
|
||||
@@ -10,10 +10,16 @@
|
||||
# rather than proceed when the sidecar never becomes idle. Once started, an
|
||||
# operation the sidecar no longer lists (its action log lives in memory and a
|
||||
# restart empties it) has to fail the Pod too, since the Job has no deadline.
|
||||
# A Job retry must not run a restore twice: it takes over the restore_remote
|
||||
# the attempt before it left running, and a few lost polls do not end an
|
||||
# A Job retry must not start its operation twice: it takes over the
|
||||
# restore_remote, or the create_remote of the archive the controller named,
|
||||
# that the attempt before it left running, and a few lost polls do not end an
|
||||
# attempt in the first place.
|
||||
#
|
||||
# Which archive a restore reads back is decided by the script's own jq
|
||||
# selection over list/remote, so it is exercised against several archives
|
||||
# rather than string-matched: exactly the archive the Backup recorded, or for
|
||||
# a Backup that records none, the newest one stamped no later than its takenAt.
|
||||
#
|
||||
# Run from the repo root with make bats-unit-tests. The rendered script is run
|
||||
# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production
|
||||
# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects.
|
||||
@@ -22,18 +28,25 @@ load test_helper
|
||||
|
||||
CHART=packages/system/backupstrategy-controller
|
||||
|
||||
# Render the shipped strategy's in-Pod script into $1 with the controller-render
|
||||
# placeholders substituted ($2 mode, $3 application name). An empty render, or
|
||||
# a placeholder left unsubstituted, is an early failure: these tests EXECUTE it.
|
||||
# Render the shipped strategy's in-Pod script into $1 and substitute the
|
||||
# controller-render placeholders the way the driver fills them: $2 mode, $3
|
||||
# app (release and source application), $4 the archive name the controller
|
||||
# chose or the Backup recorded (empty, the default, for none), $5 the Backup
|
||||
# name, $6 its takenAt in the archive-name layout (default: later than every
|
||||
# archive the stub lists). An empty render fails early, as does a placeholder
|
||||
# left unsubstituted: these tests EXECUTE it.
|
||||
render_script() {
|
||||
out=$1; mode=$2; app=$3
|
||||
out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z}
|
||||
helm template t "$CHART" -n cozy-backup-controller \
|
||||
--set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \
|
||||
| yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \
|
||||
| sed -e "s#{{ .Mode }}#${mode}#g" \
|
||||
-e "s#{{ .Release.Name }}#${app}#g" \
|
||||
-e "s#{{ .Release.Namespace }}#tenant-x#g" \
|
||||
-e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" > "$out"
|
||||
-e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" \
|
||||
-e "s#{{ .Backup.Name }}#${backup}#g" \
|
||||
-e "s#{{ .ArchiveName | default \"\" }}#${archive}#g" \
|
||||
-e "s#{{ .Backup.TakenAt | default \"\" }}#${takenat}#g" > "$out"
|
||||
[ -s "$out" ] || return 1
|
||||
if grep -q '{{' "$out"; then
|
||||
echo "rendered script still holds a controller placeholder" >&2
|
||||
@@ -49,13 +62,17 @@ render_script() {
|
||||
# busy a number: that many GET /backup/actions answers still list
|
||||
# an operation of someone else "in progress"
|
||||
# op the command line of the sidecar's own operation, written
|
||||
# by an accepted POST or seeded by a test as the operation
|
||||
# by an accepted POST the way the sidecar logs it ("--resume"
|
||||
# before the name of a create_remote asking for it), or
|
||||
# seeded by a test as the operation
|
||||
# an earlier attempt of the same Job started
|
||||
# op_left how many more GET /backup/actions answers list op as
|
||||
# "in progress" before it ends with op_result
|
||||
# op_duration op_left given to an operation a POST starts (default 0)
|
||||
# op_result the status op ends with (default success)
|
||||
# prior extra action-log rows, verbatim, older than op
|
||||
# remote present: the list/remote answer, verbatim, in place of the
|
||||
# one archive listed by default
|
||||
# restarted present: the action log no longer lists op, as after the
|
||||
# sidecar restarted and lost its in-memory log
|
||||
# fail_on_post a POST that is accepted makes that many following
|
||||
@@ -94,7 +111,10 @@ case "\$method \$url" in
|
||||
"POST "*)
|
||||
if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi
|
||||
case "\$url" in
|
||||
*/create_remote\?name=*) cmd="create_remote \${url##*=}" ;;
|
||||
*/create_remote\?name=*)
|
||||
name=\${url#*name=}; name=\${name%%&*}
|
||||
case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac
|
||||
;;
|
||||
*/restore_remote/*) cmd="restore_remote \${url##*/}" ;;
|
||||
*) exit 22 ;;
|
||||
esac
|
||||
@@ -130,7 +150,9 @@ case "\$method \$url" in
|
||||
if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi
|
||||
;;
|
||||
"GET "*/backup/list/remote)
|
||||
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
|
||||
if [ -e "\$state/remote" ]; then cat "\$state/remote"; else
|
||||
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
@@ -142,6 +164,17 @@ STUB
|
||||
chmod 0755 "$bin/curl" "$bin/sleep"
|
||||
}
|
||||
|
||||
# One list/remote row per argument, "name" or "name|desc"; created mirrors
|
||||
# the name's timestamp.
|
||||
remote_rows() {
|
||||
for row in "$@"; do
|
||||
name=${row%%|*}; desc="tar, regular"
|
||||
if [ "$row" != "$name" ]; then desc=${row#*|}; fi
|
||||
ts=${name##*-}
|
||||
printf '{"name":"%s","created":"%s","location":"remote","required":"","desc":"%s"}\n' "$name" "$ts" "$desc"
|
||||
done
|
||||
}
|
||||
|
||||
# The number of operations the sidecar accepted.
|
||||
accepted() {
|
||||
if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi
|
||||
@@ -175,7 +208,7 @@ run_script() {
|
||||
echo 3 > "$tmp/state/busy"
|
||||
render_script "$tmp/s.sh" backup myapp
|
||||
run_script "$tmp"
|
||||
grep -q '^create_remote clickhouse-myapp-' "$tmp/state/accepted"
|
||||
grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted"
|
||||
[ "$(cat "$tmp/state/busy")" = 0 ]
|
||||
}
|
||||
|
||||
@@ -321,3 +354,144 @@ run_script() {
|
||||
if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi
|
||||
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi
|
||||
}
|
||||
|
||||
@test "restore reads back the archive the Backup recorded, not the newest one" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
|
||||
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
|
||||
clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote"
|
||||
run_script "$tmp"
|
||||
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
}
|
||||
|
||||
@test "restore fails naming the recorded archive when remote storage lacks it" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
|
||||
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
|
||||
out=""
|
||||
if out=$(run_script "$tmp" 2>&1); then
|
||||
echo "FAIL: restore succeeded without the recorded archive: $out"
|
||||
false
|
||||
fi
|
||||
printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage'
|
||||
if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then
|
||||
echo "FAIL: restored another archive: $(cat "$tmp/state/requests")"
|
||||
false
|
||||
fi
|
||||
}
|
||||
|
||||
@test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
|
||||
# Each decoy sorts after the expected archive and is ruled out by one
|
||||
# filter: broken, not a bare timestamp after the prefix, or stamped after
|
||||
# takenAt.
|
||||
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
|
||||
"clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \
|
||||
clickhouse-myapp-20260304T050710Z-copy \
|
||||
clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
|
||||
out=$(run_script "$tmp" 2>&1)
|
||||
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name'
|
||||
}
|
||||
|
||||
@test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
|
||||
remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote"
|
||||
out=""
|
||||
if out=$(run_script "$tmp" 2>&1); then
|
||||
echo "FAIL: restore succeeded with every archive newer than takenAt: $out"
|
||||
false
|
||||
fi
|
||||
printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z'
|
||||
if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then
|
||||
echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")"
|
||||
false
|
||||
fi
|
||||
}
|
||||
|
||||
@test "backup writes the archive under the name the controller chose" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests"
|
||||
}
|
||||
|
||||
@test "backup falls back to a timestamped name when the controller supplies none" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
render_script "$tmp/s.sh" backup myapp ""
|
||||
run_script "$tmp"
|
||||
grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests"
|
||||
}
|
||||
|
||||
# The sidecar answers a second operation with 423 while one runs, so a retry
|
||||
# that POSTed over its predecessor's upload would burn every attempt left.
|
||||
@test "a retried backup takes over the upload of its archive the sidecar is still running" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op"
|
||||
echo 3 > "$tmp/state/op_left"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
if grep -q '^POST ' "$tmp/state/requests"; then
|
||||
echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")"
|
||||
false
|
||||
fi
|
||||
[ "$(cat "$tmp/state/op_left")" = 0 ]
|
||||
}
|
||||
|
||||
@test "a taken-over backup that ends in error fails the Pod" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op"
|
||||
echo 2 > "$tmp/state/op_left"
|
||||
echo error > "$tmp/state/op_result"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi
|
||||
if grep -q '^POST ' "$tmp/state/requests"; then
|
||||
echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")"
|
||||
false
|
||||
fi
|
||||
}
|
||||
|
||||
@test "a retried backup after a failed attempt resumes the same archive" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests"
|
||||
}
|
||||
|
||||
@test "backup does not take over an upload running for another archive" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op"
|
||||
echo 3 > "$tmp/state/op_left"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted"
|
||||
}
|
||||
|
||||
# The action log is never trimmed, so a finished row of the same archive may
|
||||
# be an earlier run's and is not taken for this attempt's result.
|
||||
@test "backup starts create_remote again when the log holds a finished one of the same archive" {
|
||||
for status in success error cancel; do
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier upload must not stand in for this one"; false; fi
|
||||
done
|
||||
}
|
||||
@@ -488,6 +488,33 @@ pod_cidrs_of() {
|
||||
rm -rf "$STATE"
|
||||
}
|
||||
|
||||
@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" {
|
||||
init_state
|
||||
add_ip web-0.tenant-c ovn-default 10.244.1.135
|
||||
add_ip web-1.tenant-c ovn-default 10.244.4.135
|
||||
add_ip db-0.tenant-c ovn-default 10.244.9.9
|
||||
# Cilium copies every Node label onto the CiliumNode, which takes six real
|
||||
# ones past the 128 KiB Linux allows a single argument. One annotation stands
|
||||
# in for the labels and keeps the list over that limit even printed compact.
|
||||
for i in 1 2 3 4 5 6; do
|
||||
add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135"
|
||||
jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp"
|
||||
mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json"
|
||||
done
|
||||
[ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ]
|
||||
"$SCRIPT" > "$STATE/out" 2>&1
|
||||
# Each pod is named with the node whose Ingress IP it holds, and a pod on an
|
||||
# address that is no node's Ingress IP is not named at all.
|
||||
grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out"
|
||||
grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out"
|
||||
if grep -q "db-0.tenant-c" "$STATE/out"; then
|
||||
cat "$STATE/out"
|
||||
false
|
||||
fi
|
||||
[ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ]
|
||||
rm -rf "$STATE"
|
||||
}
|
||||
|
||||
@test "a cluster without kube-ovn skips the exclusion and still moves its nodes" {
|
||||
init_state
|
||||
rm "$STATE/subnet.json"
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env bats
|
||||
# -----------------------------------------------------------------------------
|
||||
# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[]
|
||||
# list inside its container command.
|
||||
#
|
||||
# The Job's script reaches the container as one `sh -c` argument, and Linux caps
|
||||
# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per
|
||||
# user written into that script, about 240 users take it past the cap: the
|
||||
# container cannot start, the post-install/post-upgrade hook fails, and the
|
||||
# tenant cluster's HelmRelease stays unready. The bindings reach the Job through
|
||||
# a mounted ConfigMap instead, so the script is the same size for any list.
|
||||
#
|
||||
# The last three tests execute the rendered script against a fake kubectl, since
|
||||
# no assertion on the script's text says what the prune step does with the list:
|
||||
# a condition forced either way passes every helm-unittest case.
|
||||
#
|
||||
# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
load test_helper
|
||||
|
||||
USERS=300
|
||||
|
||||
# render_users <dir> <count> renders the chart in System mode with <count>
|
||||
# generated users and leaves one JSON document per object in <dir>/rendered.json.
|
||||
render_users() {
|
||||
jq -n --argjson n "$2" '
|
||||
{oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \
|
||||
> "$1/users.yaml"
|
||||
helm template bindings packages/apps/kubernetes \
|
||||
--namespace tenant-test \
|
||||
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
|
||||
--values "$1/users.yaml" \
|
||||
> "$1/rendered.yaml"
|
||||
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
|
||||
}
|
||||
|
||||
# render_job <dir> <users JSON> renders the chart in CustomConfig mode and unpacks
|
||||
# what the bootstrap Job runs and mounts into <dir>: script.sh is its `sh -c`
|
||||
# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an
|
||||
# empty value an empty file, as the kubelet writes it.
|
||||
render_job() {
|
||||
jq -n --argjson users "$2" '
|
||||
{_cluster: {"oidc-enabled": "false"},
|
||||
oidc: {mode: "CustomConfig",
|
||||
customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"},
|
||||
users: $users}}' > "$1/values.yaml"
|
||||
helm template bindings packages/apps/kubernetes \
|
||||
--namespace tenant-test \
|
||||
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
|
||||
--values "$1/values.yaml" \
|
||||
> "$1/rendered.yaml"
|
||||
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
|
||||
jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
|
||||
| .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh"
|
||||
mkdir -p "$1/bindings"
|
||||
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
|
||||
| .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml"
|
||||
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
|
||||
| .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt"
|
||||
}
|
||||
|
||||
# fake_kubectl <dir> <names> puts a kubectl first on PATH that logs every call and
|
||||
# answers the script's list of existing bindings with <names>.
|
||||
fake_kubectl() {
|
||||
mkdir -p "$1/bin"
|
||||
printf '%s' "$2" > "$1/existing"
|
||||
cat > "$1/bin/kubectl" <<'KEOF'
|
||||
#!/bin/sh
|
||||
echo "$*" >> "$KLOG"
|
||||
case "$*" in
|
||||
*"get clusterrolebindings"*) cat "$EXISTING" ;;
|
||||
esac
|
||||
KEOF
|
||||
chmod +x "$1/bin/kubectl"
|
||||
}
|
||||
|
||||
# run_job <dir> runs the unpacked script as the container does and leaves its
|
||||
# status in $rc and its output in <dir>/out. Run by bash, which has the pipefail
|
||||
# the script sets and which is what the unit lane runs under; the container's own
|
||||
# shell is not exercised.
|
||||
run_job() {
|
||||
: > "$1/kubectl.log"
|
||||
echo kubeconfig > "$1/kubeconfig"
|
||||
rc=0
|
||||
env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \
|
||||
KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \
|
||||
OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \
|
||||
bash "$1/script.sh" > "$1/out" 2>&1 || rc=$?
|
||||
}
|
||||
|
||||
@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" {
|
||||
dir=$(mktemp -d)
|
||||
render_users "$dir" "$USERS"
|
||||
jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
|
||||
| .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script"
|
||||
size=$(wc -c < "$dir/script")
|
||||
[ "$size" -gt 0 ]
|
||||
[ "$size" -lt 131072 ]
|
||||
if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then
|
||||
echo "the Job script still carries users[] entries" >&2
|
||||
false
|
||||
fi
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" {
|
||||
dir=$(mktemp -d)
|
||||
render_users "$dir" "$USERS"
|
||||
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
|
||||
| .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml"
|
||||
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
|
||||
| .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt"
|
||||
[ -s "$dir/crbs.yaml" ]
|
||||
yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json"
|
||||
|
||||
[ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ]
|
||||
[ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ]
|
||||
[ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ]
|
||||
|
||||
# The name is what the prune step and every binding already in a tenant cluster
|
||||
# go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of
|
||||
# sha256("<release>-<email>").
|
||||
for i in 0 1 6 7; do
|
||||
email="user$i@example.org"
|
||||
name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)"
|
||||
[ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ]
|
||||
grep -q -x -F "$name" "$dir/desired.txt"
|
||||
done
|
||||
|
||||
# The list the prune step compares against is exactly the applied names.
|
||||
[ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ]
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
@test "the Job applies the mounted bindings and deletes only the bindings no user has" {
|
||||
dir=$(mktemp -d)
|
||||
render_job "$dir" '[{"email":"alice@example.org","role":"admin"},{"email":"bob@example.org","role":"view"}]'
|
||||
alice="cozystack-oidc-$(printf '%s' 'bindings-alice@example.org' | sha256sum | cut -c1-16)"
|
||||
bob="cozystack-oidc-$(printf '%s' 'bindings-bob@example.org' | sha256sum | cut -c1-16)"
|
||||
fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef"
|
||||
run_job "$dir"
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
cat "$dir/out"
|
||||
false
|
||||
fi
|
||||
grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log"
|
||||
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log"
|
||||
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log"
|
||||
if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then
|
||||
echo "the Job deleted a binding a user still has" >&2
|
||||
false
|
||||
fi
|
||||
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
@test "a release with no users applies nothing and deletes every binding it left behind" {
|
||||
dir=$(mktemp -d)
|
||||
render_job "$dir" '[]'
|
||||
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
|
||||
run_job "$dir"
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
cat "$dir/out"
|
||||
false
|
||||
fi
|
||||
if grep -q -F -- ' apply ' "$dir/kubectl.log"; then
|
||||
echo "the Job ran kubectl apply with no bindings to apply" >&2
|
||||
false
|
||||
fi
|
||||
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
@test "the Job stops before deleting anything when the desired list is not mounted" {
|
||||
dir=$(mktemp -d)
|
||||
render_job "$dir" '[{"email":"alice@example.org","role":"admin"}]'
|
||||
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
|
||||
rm "$dir/bindings/desired.txt"
|
||||
run_job "$dir"
|
||||
[ "$rc" -ne 0 ]
|
||||
if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then
|
||||
cat "$dir/kubectl.log"
|
||||
false
|
||||
fi
|
||||
rm -rf "$dir"
|
||||
}
|
||||
@@ -2,7 +2,9 @@
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
@@ -43,6 +45,18 @@ const (
|
||||
// were in effect at backup time.
|
||||
altinityParamPrefix = "altinity.strategy.backups.cozystack.io/parameter/"
|
||||
|
||||
// altinityArchiveMetaKey records, on the Backup, the clickhouse-backup
|
||||
// archive its Job wrote, as named by altinityArchiveName. A restore reads
|
||||
// back exactly that archive: the newest one under the release's prefix
|
||||
// may belong to a later Backup.
|
||||
altinityArchiveMetaKey = "altinity.strategy.backups.cozystack.io/archive"
|
||||
|
||||
// altinityArchiveTimeLayout is the UTC timestamp that ends an archive
|
||||
// name. It is fixed-width, so names sharing a prefix sort chronologically
|
||||
// as plain strings, which is what a restore of a Backup that records no
|
||||
// archive relies on.
|
||||
altinityArchiveTimeLayout = "20060102T150405Z"
|
||||
|
||||
// Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG /
|
||||
// Velero strategy cadence so behaviour is uniform across drivers.
|
||||
altinityPollInterval = 5 * time.Second
|
||||
@@ -102,14 +116,37 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// altinityArchiveName mints the archive name for one BackupJob from its
|
||||
// persisted StartedAt, so every reconcile and every Pod retry of the Job
|
||||
// writes the same archive. It keeps the clickhouse-<release>-<timestamp>
|
||||
// shape, because a Backup that records no archive is restored by the
|
||||
// timestamp in that shape.
|
||||
func altinityArchiveName(j *backupsv1alpha1.BackupJob) string {
|
||||
return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout))
|
||||
}
|
||||
|
||||
// altinityJobCarriesArchive reports whether the Job's Pod was rendered with
|
||||
// the archive name. A Job created by a controller that predates archive
|
||||
// names, or rendered from a strategy that names its own archives, wrote an
|
||||
// archive under another name, and recording this one would point the Backup
|
||||
// at an archive that does not exist.
|
||||
func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool {
|
||||
spec, err := json.Marshal(job.Spec.Template.Spec)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Contains(spec, []byte(archive))
|
||||
}
|
||||
|
||||
// renderAltinityTemplate runs the strategy's PodTemplateSpec through the
|
||||
// repository's text/template engine with the same context shape as the CNPG
|
||||
// driver: every string field is templated against the application object,
|
||||
// the release shorthand, the run mode, and the resolved parameters.
|
||||
// the release shorthand, the run mode, the archive name, and the resolved
|
||||
// parameters.
|
||||
func renderAltinityTemplate(
|
||||
tmpl corev1.PodTemplateSpec,
|
||||
app map[string]any,
|
||||
releaseName, releaseNamespace, mode string,
|
||||
releaseName, releaseNamespace, mode, archiveName string,
|
||||
parameters map[string]string,
|
||||
backup *backupsv1alpha1.Backup,
|
||||
) (*corev1.PodTemplateSpec, error) {
|
||||
@@ -119,8 +156,9 @@ func renderAltinityTemplate(
|
||||
"Name": releaseName,
|
||||
"Namespace": releaseNamespace,
|
||||
},
|
||||
"Mode": mode,
|
||||
"Parameters": parameters,
|
||||
"Mode": mode,
|
||||
"ArchiveName": archiveName,
|
||||
"Parameters": parameters,
|
||||
}
|
||||
if backup != nil {
|
||||
// Expose ApplicationRef so restore strategies can reference the
|
||||
@@ -143,6 +181,9 @@ func renderAltinityTemplate(
|
||||
"Kind": backup.Spec.ApplicationRef.Kind,
|
||||
"Name": backup.Spec.ApplicationRef.Name,
|
||||
},
|
||||
// In the archive-name layout, so a strategy can compare it with
|
||||
// the timestamp an archive name ends in.
|
||||
"TakenAt": backup.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout),
|
||||
}
|
||||
}
|
||||
return template.Template(&tmpl, ctxMap)
|
||||
@@ -209,12 +250,14 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
archive := altinityArchiveName(j)
|
||||
rendered, err := renderAltinityTemplate(
|
||||
strategy.Spec.Template,
|
||||
app,
|
||||
j.Spec.ApplicationRef.Name,
|
||||
j.Namespace,
|
||||
altinityModeBackup,
|
||||
archive,
|
||||
resolved.Parameters,
|
||||
nil,
|
||||
)
|
||||
@@ -239,7 +282,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
|
||||
if j.Status.BackupRef != nil {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved)
|
||||
if !altinityJobCarriesArchive(batchJob, archive) {
|
||||
archive = ""
|
||||
}
|
||||
artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive)
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
|
||||
}
|
||||
@@ -411,16 +457,20 @@ func jobFailureMessage(j *batchv1.Job) string {
|
||||
}
|
||||
|
||||
// createAltinityBackupArtifact materialises a Cozystack Backup carrying the
|
||||
// strategy reference and the BackupClassStrategy parameters in effect at
|
||||
// backup time. Parameters round-trip via DriverMetadata under
|
||||
// altinityParamPrefix so a later RestoreJob can re-render the strategy
|
||||
// template against the same values.
|
||||
// strategy reference, the archive the Job wrote (when known) and the
|
||||
// BackupClassStrategy parameters in effect at backup time. Parameters
|
||||
// round-trip via DriverMetadata under altinityParamPrefix so a later
|
||||
// RestoreJob can re-render the strategy template against the same values.
|
||||
func (r *BackupJobReconciler) createAltinityBackupArtifact(
|
||||
ctx context.Context,
|
||||
j *backupsv1alpha1.BackupJob,
|
||||
resolved *ResolvedBackupConfig,
|
||||
archive string,
|
||||
) (*backupsv1alpha1.Backup, error) {
|
||||
driverMD := map[string]string{}
|
||||
if archive != "" {
|
||||
driverMD[altinityArchiveMetaKey] = archive
|
||||
}
|
||||
for k, v := range resolved.Parameters {
|
||||
driverMD[altinityParamPrefix+k] = v
|
||||
}
|
||||
@@ -459,6 +509,13 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact(
|
||||
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
|
||||
return nil, getErr
|
||||
}
|
||||
// A Backup outlives its BackupJob, so a BackupJob re-created under a
|
||||
// used name meets the earlier run's Backup here. Only one recording
|
||||
// this run's archive was created by an earlier reconcile of this run.
|
||||
if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive {
|
||||
return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote",
|
||||
backup.Namespace, backup.Name, got, archive)
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
return backup, nil
|
||||
@@ -616,6 +673,7 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
|
||||
targetAppName,
|
||||
targetNamespace,
|
||||
altinityModeRestore,
|
||||
backup.Spec.DriverMetadata[altinityArchiveMetaKey],
|
||||
backupParameters(backup),
|
||||
backup,
|
||||
)
|
||||
|
||||
@@ -3,7 +3,9 @@ package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
@@ -697,6 +699,7 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
|
||||
DriverMetadata: map[string]string{
|
||||
altinityParamPrefix + "bucketName": "b1",
|
||||
altinityParamPrefix + "endpoint": "https://s3",
|
||||
altinityArchiveMetaKey: "clickhouse-ch-test-20260304T050607Z",
|
||||
"job.batch/name": "ignored",
|
||||
"": "ignored",
|
||||
},
|
||||
@@ -706,7 +709,326 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
|
||||
if got["bucketName"] != "b1" || got["endpoint"] != "https://s3" {
|
||||
t.Errorf("backupParameters = %v", got)
|
||||
}
|
||||
if _, ok := got["job.batch/name"]; ok {
|
||||
if len(got) != 2 {
|
||||
t.Errorf("non-parameter key leaked through: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// newAltinityArchiveStrategy renders the archive name and the Backup's
|
||||
// takenAt where a test can read them back off the Job.
|
||||
func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity {
|
||||
s := newAltinityStrategy(name)
|
||||
s.Spec.Template.Spec.Containers[0].Args = []string{
|
||||
"--mode={{ .Mode }}",
|
||||
"--archive={{ .ArchiveName }}",
|
||||
"--taken-at={{ .Backup.TakenAt }}",
|
||||
"--bucket={{ .Parameters.bucketName }}",
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) {
|
||||
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600)))
|
||||
j := &backupsv1alpha1.BackupJob{
|
||||
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
|
||||
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
|
||||
}
|
||||
if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want {
|
||||
t.Errorf("altinityArchiveName = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob
|
||||
// through Job creation and completion in separate reconciles. The archive
|
||||
// name has to come out of the persisted StartedAt both times: the Pod is
|
||||
// told it on the first, the Backup records it on the second, and a restore
|
||||
// later reads back exactly that archive.
|
||||
func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) {
|
||||
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
||||
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
||||
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
|
||||
backupJob := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
||||
BackupClassName: "clickhouse-backup",
|
||||
},
|
||||
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
||||
}
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
||||
Kind: strategyv1alpha1.AltinityStrategyKind,
|
||||
Name: "clickhouse-strategy",
|
||||
},
|
||||
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
||||
}
|
||||
r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
|
||||
ctx := context.Background()
|
||||
const want = "clickhouse-ch-test-20260304T050607Z"
|
||||
|
||||
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
||||
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
|
||||
}
|
||||
job := &batchv1.Job{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
|
||||
t.Fatalf("get backup Job: %v", err)
|
||||
}
|
||||
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want {
|
||||
t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, want)
|
||||
}
|
||||
|
||||
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
|
||||
if err := r.Status().Update(ctx, job); err != nil {
|
||||
t.Fatalf("mark backup Job complete: %v", err)
|
||||
}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
|
||||
t.Fatalf("refresh BackupJob: %v", err)
|
||||
}
|
||||
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
||||
t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
|
||||
}
|
||||
|
||||
created := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
|
||||
t.Fatalf("get Backup: %v", err)
|
||||
}
|
||||
if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want {
|
||||
t.Errorf("Backup records archive %q, want %q", got, want)
|
||||
}
|
||||
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
|
||||
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose
|
||||
// Pod never carried the driver's archive name: one created before the
|
||||
// upgrade that introduced the names, or rendered from a strategy that names
|
||||
// its own archives. Recording the name would send the restore after an
|
||||
// archive nobody wrote; recording none leaves it the fallback for Backups
|
||||
// without a name.
|
||||
func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) {
|
||||
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
||||
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
||||
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
|
||||
backupJob := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
||||
BackupClassName: "clickhouse-backup",
|
||||
},
|
||||
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
||||
}
|
||||
completedK8sJob := &batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: jobNameForBackupJob(backupJob), Namespace: backupJob.Namespace},
|
||||
Spec: batchv1.JobSpec{
|
||||
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
Containers: []corev1.Container{{
|
||||
Name: "clickhouse-backup",
|
||||
Image: "altinity/clickhouse-backup:test",
|
||||
Args: []string{`target="clickhouse-ch-test-$(date -u +%Y%m%dT%H%M%SZ)"`},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Status: batchv1.JobStatus{
|
||||
Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}},
|
||||
},
|
||||
}
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
||||
Kind: strategyv1alpha1.AltinityStrategyKind,
|
||||
Name: "clickhouse-strategy",
|
||||
},
|
||||
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
||||
}
|
||||
r, _, _ := newAltinityTestEnv(t, app,
|
||||
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob))
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
||||
t.Fatalf("reconcileAltinity() error = %v", err)
|
||||
}
|
||||
created := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
|
||||
t.Fatalf("get Backup: %v", err)
|
||||
}
|
||||
if got, ok := created.Spec.DriverMetadata[altinityArchiveMetaKey]; ok {
|
||||
t.Errorf("Backup records archive %q its Job was never given", got)
|
||||
}
|
||||
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
|
||||
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileAltinity_BackupNameAlreadyTaken covers a completed Job whose
|
||||
// Backup name is already in use. A Backup outlives its BackupJob, so a
|
||||
// BackupJob re-created under an earlier run's name finds that run's Backup,
|
||||
// and claiming it would point the new run at an archive it did not write.
|
||||
// A Backup recording this run's own archive is the one an earlier reconcile
|
||||
// created before its status write was lost, and is adopted.
|
||||
func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) {
|
||||
const thisRun = "clickhouse-ch-test-20260305T050607Z"
|
||||
cases := []struct {
|
||||
name string
|
||||
existing string // archive the existing Backup records; empty for none
|
||||
wantSucceeded bool
|
||||
}{
|
||||
{name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"},
|
||||
{name: "earlier run's Backup records no archive", existing: ""},
|
||||
{name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
||||
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
||||
startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC))
|
||||
backupJob := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
||||
BackupClassName: "clickhouse-backup",
|
||||
},
|
||||
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
||||
}
|
||||
driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
|
||||
if tc.existing != "" {
|
||||
driverMD[altinityArchiveMetaKey] = tc.existing
|
||||
}
|
||||
existing := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: backupJob.Name, Namespace: backupJob.Namespace},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
||||
StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
||||
Kind: strategyv1alpha1.AltinityStrategyKind,
|
||||
Name: "clickhouse-strategy",
|
||||
},
|
||||
TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 22, 0, time.UTC)),
|
||||
DriverMetadata: driverMD,
|
||||
},
|
||||
}
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
||||
Kind: strategyv1alpha1.AltinityStrategyKind,
|
||||
Name: "clickhouse-strategy",
|
||||
},
|
||||
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
||||
}
|
||||
r, _, _ := newAltinityTestEnv(t, app,
|
||||
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, existing))
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
||||
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
|
||||
}
|
||||
job := &batchv1.Job{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
|
||||
t.Fatalf("get backup Job: %v", err)
|
||||
}
|
||||
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+thisRun {
|
||||
t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, thisRun)
|
||||
}
|
||||
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
|
||||
if err := r.Status().Update(ctx, job); err != nil {
|
||||
t.Fatalf("mark backup Job complete: %v", err)
|
||||
}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
|
||||
t.Fatalf("refresh BackupJob: %v", err)
|
||||
}
|
||||
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
||||
t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
|
||||
}
|
||||
|
||||
updated := &backupsv1alpha1.BackupJob{}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if tc.wantSucceeded {
|
||||
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded {
|
||||
t.Errorf("phase = %q (%s), want Succeeded", updated.Status.Phase, updated.Status.Message)
|
||||
}
|
||||
if updated.Status.BackupRef == nil || updated.Status.BackupRef.Name != existing.Name {
|
||||
t.Errorf("BackupRef = %v, want %s", updated.Status.BackupRef, existing.Name)
|
||||
}
|
||||
} else {
|
||||
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Errorf("phase = %q, want Failed: the run would claim a Backup recording %q instead of %q", updated.Status.Phase, tc.existing, thisRun)
|
||||
}
|
||||
if updated.Status.BackupRef != nil {
|
||||
t.Errorf("BackupRef = %v, want none", updated.Status.BackupRef)
|
||||
}
|
||||
if !strings.Contains(updated.Status.Message, thisRun) {
|
||||
t.Errorf("failure message %q does not name the archive this run wrote", updated.Status.Message)
|
||||
}
|
||||
}
|
||||
kept := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(existing), kept); err != nil {
|
||||
t.Fatalf("get existing Backup: %v", err)
|
||||
}
|
||||
if got := kept.Spec.DriverMetadata[altinityArchiveMetaKey]; got != tc.existing {
|
||||
t.Errorf("existing Backup now records archive %q, want %q left in place", got, tc.existing)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
archive string // recorded on the Backup; empty for a Backup taken before names were recorded
|
||||
wantArchive string
|
||||
}{
|
||||
{name: "recorded archive", archive: "clickhouse-ch-test-20260304T050607Z", wantArchive: "--archive=clickhouse-ch-test-20260304T050607Z"},
|
||||
{name: "no recorded archive", archive: "", wantArchive: "--archive="},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
targetApp := newAltinityClickHouseApp("ch-test", "tenant-test")
|
||||
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
||||
driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
|
||||
if tc.archive != "" {
|
||||
driverMD[altinityArchiveMetaKey] = tc.archive
|
||||
}
|
||||
now := metav1.Now()
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
||||
StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
||||
Kind: strategyv1alpha1.AltinityStrategyKind,
|
||||
Name: "clickhouse-strategy",
|
||||
},
|
||||
TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 22, 0, time.FixedZone("UTC+3", 3*3600))),
|
||||
DriverMetadata: driverMD,
|
||||
},
|
||||
}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
||||
}
|
||||
_, rr, _ := newAltinityTestEnv(t, targetApp,
|
||||
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
|
||||
t.Fatalf("reconcileAltinityRestore: %v", err)
|
||||
}
|
||||
job := &batchv1.Job{}
|
||||
if err := rr.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForRestoreJob(restoreJob)}, job); err != nil {
|
||||
t.Fatalf("get restore Job: %v", err)
|
||||
}
|
||||
args := job.Spec.Template.Spec.Containers[0].Args
|
||||
want := []string{"--mode=restore", tc.wantArchive, "--taken-at=20260304T050622Z", "--bucket=ch-bucket"}
|
||||
for i, w := range want {
|
||||
if i >= len(args) || args[i] != w {
|
||||
t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
)
|
||||
|
||||
// artifactSizeProbeTimeout bounds the object-store round trips a driver
|
||||
// makes to measure Backup.status.artifact.sizeBytes. The probe runs inside
|
||||
// the reconcile that creates the Backup, and the BackupJob controller has a
|
||||
// single worker, so an unreachable endpoint stalls every other BackupJob for
|
||||
// this long. A variable so tests can shorten it.
|
||||
var artifactSizeProbeTimeout = 10 * time.Second
|
||||
|
||||
type s3ObjectInfo struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
type s3ObjectLister interface {
|
||||
ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error)
|
||||
}
|
||||
|
||||
// s3Connection is what a driver resolves from its rendered strategy to
|
||||
// read the bucket with the same identity the backup was written with.
|
||||
type s3Connection struct {
|
||||
// Endpoint is host[:port], without a scheme.
|
||||
Endpoint string
|
||||
Secure bool
|
||||
Region string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
SessionToken string
|
||||
// CAPEM is an endpoint CA bundle. It is trusted on top of the system
|
||||
// roots, or instead of them when CAOnly is set: the probe must trust
|
||||
// what the writer of the backup trusted, no more.
|
||||
CAPEM []byte
|
||||
CAOnly bool
|
||||
}
|
||||
|
||||
// objectStore reads back what a backup wrote: the Secrets it was written
|
||||
// with and the bucket itself. It is a value rather than reconciler methods
|
||||
// so that the restore path, which has a reconciler of its own, can read
|
||||
// the same objects.
|
||||
type objectStore struct {
|
||||
// reader reads Secrets, which the manager's client fetches uncached,
|
||||
// and lists operator Backups, which it serves from the cache.
|
||||
reader client.Reader
|
||||
// newLister overrides the S3 client; nil means a real minio client.
|
||||
newLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
}
|
||||
|
||||
func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
if s.newLister != nil {
|
||||
return s.newLister(conn)
|
||||
}
|
||||
return newMinioObjectLister(conn)
|
||||
}
|
||||
|
||||
func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) {
|
||||
secret := &corev1.Secret{}
|
||||
if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil {
|
||||
return "", err
|
||||
}
|
||||
v, ok := secret.Data[key]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("secret %s/%s has no key %q", namespace, name, key)
|
||||
}
|
||||
return string(v), nil
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
return r.objectStore().lister(conn)
|
||||
}
|
||||
|
||||
type minioObjectLister struct {
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken),
|
||||
Region: conn.Region,
|
||||
Secure: conn.Secure,
|
||||
}
|
||||
if conn.Secure && len(conn.CAPEM) > 0 {
|
||||
// Start from minio's transport to keep its proxy and timeout settings.
|
||||
tr, err := minio.DefaultTransport(true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts.Transport = tr
|
||||
}
|
||||
// minio.New refuses an endpoint with a path prefix, which it cannot
|
||||
// address, and accepts a bare trailing slash.
|
||||
client, err := minio.New(conn.Endpoint, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &minioObjectLister{client: client}, nil
|
||||
}
|
||||
|
||||
// endpointTrustPool returns the roots to verify the endpoint against: the
|
||||
// CA bundle on top of system, or the CA bundle alone with CAOnly. A nil
|
||||
// system means the platform's roots.
|
||||
func endpointTrustPool(conn s3Connection, system *x509.CertPool) (*x509.CertPool, error) {
|
||||
pool := system
|
||||
switch {
|
||||
case conn.CAOnly:
|
||||
pool = x509.NewCertPool()
|
||||
case pool == nil:
|
||||
var err error
|
||||
if pool, err = x509.SystemCertPool(); err != nil {
|
||||
pool = x509.NewCertPool()
|
||||
}
|
||||
}
|
||||
if !pool.AppendCertsFromPEM(conn.CAPEM) {
|
||||
return nil, errors.New("no PEM certificates in the endpoint CA bundle")
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
|
||||
return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}))
|
||||
}
|
||||
|
||||
// collectObjects drains a minio listing. minio-go ends a listing whose
|
||||
// context is cancelled between pages by closing the channel without an
|
||||
// error item, so a cancelled context is checked once the channel closes:
|
||||
// otherwise a timeout would pass a partial listing off as complete.
|
||||
func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3ObjectInfo, error) {
|
||||
var out []s3ObjectInfo
|
||||
for obj := range objects {
|
||||
if obj.Err != nil {
|
||||
return nil, obj.Err
|
||||
}
|
||||
out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size, LastModified: obj.LastModified})
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// errBackupNotReadBack reports that Create collided with an existing Backup
|
||||
// that could then not be read. The backup itself succeeded, so callers
|
||||
// retry the reconcile instead of failing the BackupJob.
|
||||
var errBackupNotReadBack = errors.New("the Backup already exists but could not be read back")
|
||||
|
||||
// failBackupArtifact handles an error from creating the Backup of a
|
||||
// completed backup: errBackupNotReadBack is retried through the controller's
|
||||
// backoff, with Ready=False telling why the BackupJob is still Running;
|
||||
// anything else fails the BackupJob.
|
||||
func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backupsv1alpha1.BackupJob, err error) (ctrl.Result, error) {
|
||||
if errors.Is(err, errBackupNotReadBack) {
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "BackupNotReadBack",
|
||||
Message: err.Error(),
|
||||
})
|
||||
if uerr := r.Status().Update(ctx, j); uerr != nil {
|
||||
getLogger(ctx).Info("cannot record that the Backup could not be read back", "backupjob", j.Name, "error", uerr)
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
|
||||
}
|
||||
|
||||
// existingBackup returns the Backup a previous reconcile already created
|
||||
// for the job, so a retry skips the size probe whose result it would
|
||||
// discard. It reads past the cache: a retry typically follows a Create made
|
||||
// moments earlier, which the cache may not show yet. A failed read is not
|
||||
// fatal, because the Create that follows handles an existing Backup on its
|
||||
// own; probe is false then, since the Backup may well exist and a probe
|
||||
// would only be discarded, while a read that keeps failing would rerun it on
|
||||
// every retry.
|
||||
func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (existing *backupsv1alpha1.Backup, probe bool) {
|
||||
existing = &backupsv1alpha1.Backup{}
|
||||
if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil, true
|
||||
}
|
||||
getLogger(ctx).Info("cannot check for an existing Backup; creating it without measuring its size", "backup", name, "error", err)
|
||||
return nil, false
|
||||
}
|
||||
return existing, false
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) {
|
||||
return r.objectStore().secretValue(ctx, namespace, name, key)
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
)
|
||||
|
||||
type fakeS3Lister struct {
|
||||
objects []s3ObjectInfo
|
||||
err error
|
||||
|
||||
conn s3Connection
|
||||
bucket, prefix string
|
||||
recursive bool
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
|
||||
f.calls++
|
||||
f.bucket, f.prefix, f.recursive = bucket, prefix, recursive
|
||||
return f.objects, f.err
|
||||
}
|
||||
|
||||
func s3CredsSecret(namespace, name string) *corev1.Secret {
|
||||
return &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
|
||||
Data: map[string][]byte{
|
||||
"AWS_ACCESS_KEY_ID": []byte("AKID"),
|
||||
"AWS_SECRET_ACCESS_KEY": []byte("secret"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed
|
||||
// by a fake client seeded with objs (typically the credentials Secret).
|
||||
func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler {
|
||||
t.Helper()
|
||||
c := newMariaDBStrategyTestClient(t, objs...)
|
||||
return &BackupJobReconciler{
|
||||
Client: c,
|
||||
Scheme: c.Scheme(),
|
||||
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
|
||||
l.conn = conn
|
||||
return l, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectObjects(t *testing.T) {
|
||||
listing := func() <-chan minio.ObjectInfo {
|
||||
ch := make(chan minio.ObjectInfo, 1)
|
||||
ch <- minio.ObjectInfo{Key: "a", Size: 1}
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
got, err := collectObjects(context.Background(), listing())
|
||||
if err != nil || len(got) != 1 || got[0].Size != 1 {
|
||||
t.Fatalf("collectObjects = %v, %v; want one object", got, err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err)
|
||||
}
|
||||
|
||||
errCh := make(chan minio.ObjectInfo, 1)
|
||||
errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")}
|
||||
close(errCh)
|
||||
if _, err := collectObjects(context.Background(), errCh); err == nil {
|
||||
t.Error("listing error was not returned")
|
||||
}
|
||||
}
|
||||
|
||||
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
|
||||
// Backup the cache does not show yet: the previous reconcile created it
|
||||
// moments earlier.
|
||||
func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
|
||||
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
|
||||
r := &BackupJobReconciler{
|
||||
Client: newMariaDBStrategyTestClient(t),
|
||||
APIReader: newMariaDBStrategyTestClient(t, backup),
|
||||
}
|
||||
if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
|
||||
t.Fatal("existingBackup = nil; want the Backup the API server holds")
|
||||
}
|
||||
r.APIReader = newMariaDBStrategyTestClient(t)
|
||||
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe {
|
||||
t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe)
|
||||
}
|
||||
}
|
||||
|
||||
// staleCache wraps a fake client so that Get reports NotFound for Backups,
|
||||
// the way an informer cache does right after another writer created one.
|
||||
func staleCache(t *testing.T, c client.Client) client.Client {
|
||||
t.Helper()
|
||||
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
||||
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
||||
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
|
||||
}
|
||||
return cl.Get(ctx, key, obj, opts...)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
|
||||
// the retry check runs before the Backup exists, the refetch after Create
|
||||
// collides with it.
|
||||
func notFoundOnce(t *testing.T, c client.Client) client.Client {
|
||||
t.Helper()
|
||||
missed := false
|
||||
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
||||
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
|
||||
missed = true
|
||||
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
|
||||
}
|
||||
return cl.Get(ctx, key, obj, opts...)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
|
||||
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
|
||||
return errors.New("etcdserver: request timed out")
|
||||
},
|
||||
})
|
||||
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
|
||||
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe {
|
||||
t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe)
|
||||
}
|
||||
}
|
||||
|
||||
// blockingLister answers nothing until the probe's context is done, the way
|
||||
// a black-holed endpoint does.
|
||||
type blockingLister struct{}
|
||||
|
||||
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
|
||||
// is not back well before the default timeout would have expired.
|
||||
func withShortProbeTimeout(t *testing.T, fn func()) {
|
||||
t.Helper()
|
||||
saved := artifactSizeProbeTimeout
|
||||
artifactSizeProbeTimeout = 50 * time.Millisecond
|
||||
defer func() { artifactSizeProbeTimeout = saved }()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
fn()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the probe did not give up at its timeout")
|
||||
}
|
||||
}
|
||||
|
||||
// failingReads wraps a fake client whose every Get fails, like an apiserver
|
||||
// that times out.
|
||||
func failingReads(t *testing.T, c client.Client) client.Client {
|
||||
t.Helper()
|
||||
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
|
||||
return errors.New("etcdserver: request timed out")
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestFailBackupArtifact(t *testing.T) {
|
||||
newJob := func() *backupsv1alpha1.BackupJob {
|
||||
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
|
||||
}
|
||||
|
||||
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
|
||||
j := newJob()
|
||||
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
|
||||
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
|
||||
if !errors.Is(err, errBackupNotReadBack) {
|
||||
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
|
||||
}
|
||||
persisted := &backupsv1alpha1.BackupJob{}
|
||||
if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Error("the BackupJob was failed although its Backup exists")
|
||||
}
|
||||
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" {
|
||||
t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("any other error fails the BackupJob", func(t *testing.T) {
|
||||
j := newJob()
|
||||
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
|
||||
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
|
||||
t.Fatalf("failBackupArtifact: %v", err)
|
||||
}
|
||||
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Errorf("phase = %q, want Failed", j.Status.Phase)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
|
||||
// roots" and "the CA alone". The system pool holds the endpoint's
|
||||
// certificate, and the configured CA is an unrelated one, so the endpoint
|
||||
// must stop being trusted only when CAOnly is set.
|
||||
func TestEndpointTrustPool_CAOnly(t *testing.T) {
|
||||
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
|
||||
defer endpoint.Close()
|
||||
system := x509.NewCertPool()
|
||||
system.AddCert(endpoint.Certificate())
|
||||
|
||||
// Every httptest server shares one certificate, so the unrelated CA is
|
||||
// minted here.
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
|
||||
verifies := func(caOnly bool) bool {
|
||||
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
|
||||
if err != nil {
|
||||
t.Fatalf("endpointTrustPool: %v", err)
|
||||
}
|
||||
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
|
||||
return err == nil
|
||||
}
|
||||
if !verifies(false) {
|
||||
t.Error("CA on top of the system roots: the endpoint was not trusted")
|
||||
}
|
||||
if verifies(true) {
|
||||
t.Error("CA alone: the endpoint was trusted through the system roots")
|
||||
}
|
||||
}
|
||||
|
||||
// forbidBackupCreate wraps a fake client whose Create of a Backup is
|
||||
// rejected, the way an admission webhook rejects one.
|
||||
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
|
||||
t.Helper()
|
||||
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
||||
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
||||
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
||||
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
|
||||
}
|
||||
return cl.Create(ctx, obj, opts...)
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -39,6 +39,10 @@ type BackupJobReconciler struct {
|
||||
// re-reads the BackupJob before dispatching to a driver. Wired in
|
||||
// SetupWithManager.
|
||||
APIReader client.Reader
|
||||
|
||||
// newS3ObjectLister overrides the S3 client used to measure artifact
|
||||
// sizes; nil means a real minio client. Set by unit tests.
|
||||
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
||||
)
|
||||
|
||||
// cnpgArtifactSizeBytes sums the objects barman-cloud uploaded for one base
|
||||
// backup, <destinationPath>/<serverName>/base/<backupID>/. The WAL archive
|
||||
// is not counted: it is one continuous stream shared by every base backup of
|
||||
// the server, so no part of it belongs to a single Backup. Returns 0 -
|
||||
// leaving Artifact.SizeBytes absent - when the backup ID is unknown, the
|
||||
// bucket cannot be read, or nothing is stored under the prefix.
|
||||
func (r *BackupJobReconciler) cnpgArtifactSizeBytes(
|
||||
ctx context.Context,
|
||||
namespace string,
|
||||
store *strategyv1alpha1.BarmanObjectStoreTemplate,
|
||||
serverName, backupID string,
|
||||
) int64 {
|
||||
logger := getLogger(ctx)
|
||||
if backupID == "" {
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout)
|
||||
defer cancel()
|
||||
|
||||
bucket, path, err := parseBarmanS3DestinationPath(store.DestinationPath)
|
||||
if err != nil {
|
||||
logger.Info("cannot measure CNPG backup size", "backupID", backupID, "error", err)
|
||||
return 0
|
||||
}
|
||||
conn, err := r.cnpgS3Connection(ctx, namespace, store)
|
||||
if err != nil {
|
||||
logger.Info("cannot resolve S3 credentials to measure CNPG backup size", "backupID", backupID, "error", err)
|
||||
return 0
|
||||
}
|
||||
lister, err := r.s3Lister(conn)
|
||||
if err != nil {
|
||||
logger.Info("cannot build S3 client to measure CNPG backup size", "backupID", backupID, "error", err)
|
||||
return 0
|
||||
}
|
||||
prefix := barmanBaseBackupPrefix(path, serverName, backupID)
|
||||
objects, err := lister.ListObjects(ctx, bucket, prefix, true)
|
||||
if err != nil {
|
||||
logger.Info("cannot list CNPG base backup objects to measure size", "backupID", backupID, "error", err)
|
||||
return 0
|
||||
}
|
||||
var total int64
|
||||
for _, obj := range objects {
|
||||
total += obj.Size
|
||||
}
|
||||
if total == 0 {
|
||||
logger.Info("no CNPG base backup objects found; leaving size unset", "backupID", backupID, "prefix", prefix)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace string, store *strategyv1alpha1.BarmanObjectStoreTemplate) (s3Connection, error) {
|
||||
var conn s3Connection
|
||||
if store.S3Credentials == nil {
|
||||
return conn, errors.New("barmanObjectStore has no s3Credentials")
|
||||
}
|
||||
if store.EndpointURL == "" {
|
||||
conn.Endpoint, conn.Secure = "s3.amazonaws.com", true
|
||||
} else {
|
||||
u, err := url.Parse(store.EndpointURL)
|
||||
if err != nil || u.Host == "" {
|
||||
return conn, fmt.Errorf("invalid endpointURL %q", store.EndpointURL)
|
||||
}
|
||||
// Dropping the path would probe the bucket at another location than
|
||||
// the one barman writes to.
|
||||
if strings.Trim(u.Path, "/") != "" {
|
||||
return conn, fmt.Errorf("endpointURL %q has a path, which the S3 client cannot address", store.EndpointURL)
|
||||
}
|
||||
conn.Endpoint, conn.Secure = u.Host, u.Scheme == "https"
|
||||
}
|
||||
|
||||
creds := store.S3Credentials
|
||||
accessKeyKey, secretKeyKey := creds.AccessKeyIDKey, creds.SecretAccessKeyKey
|
||||
if accessKeyKey == "" {
|
||||
accessKeyKey = defaultS3AccessKeyIDKey
|
||||
}
|
||||
if secretKeyKey == "" {
|
||||
secretKeyKey = defaultS3SecretAccessKeyKey
|
||||
}
|
||||
var err error
|
||||
if conn.AccessKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, accessKeyKey); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if conn.SecretKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, secretKeyKey); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if ca := store.EndpointCA; ca != nil && ca.SecretRef.Name != "" && conn.Secure {
|
||||
key := ca.Key
|
||||
if key == "" {
|
||||
key = defaultEndpointCAKey
|
||||
}
|
||||
pem, err := r.secretValue(ctx, namespace, ca.SecretRef.Name, key)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
// barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the
|
||||
// default trust store rather than adding to it.
|
||||
conn.CAPEM, conn.CAOnly = []byte(pem), true
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// parseBarmanS3DestinationPath splits s3://bucket/path into the bucket and
|
||||
// the key path barman-cloud prefixes every object with.
|
||||
func parseBarmanS3DestinationPath(destinationPath string) (bucket, path string, err error) {
|
||||
u, err := url.Parse(destinationPath)
|
||||
if err != nil || u.Scheme != "s3" || u.Host == "" {
|
||||
return "", "", fmt.Errorf("destinationPath %q is not an s3:// URL", destinationPath)
|
||||
}
|
||||
return u.Host, strings.Trim(u.Path, "/"), nil
|
||||
}
|
||||
|
||||
// barmanBaseBackupPrefix mirrors barman-cloud's
|
||||
// os.path.join(path, server_name, "base", backup_id), with a trailing slash
|
||||
// so one backup ID never matches another that extends it.
|
||||
func barmanBaseBackupPrefix(path, serverName, backupID string) string {
|
||||
parts := make([]string, 0, 4)
|
||||
if path != "" {
|
||||
parts = append(parts, path)
|
||||
}
|
||||
parts = append(parts, serverName, "base", backupID)
|
||||
return strings.Join(parts, "/") + "/"
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
|
||||
)
|
||||
|
||||
func newSizedBarmanObjectStore() *strategyv1alpha1.BarmanObjectStoreTemplate {
|
||||
return &strategyv1alpha1.BarmanObjectStoreTemplate{
|
||||
DestinationPath: "s3://bkt/tenant/pg/",
|
||||
EndpointURL: "https://s3.example:8443",
|
||||
S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{
|
||||
SecretRef: corev1.LocalObjectReference{Name: "creds"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestBarmanBaseBackupPrefix(t *testing.T) {
|
||||
cases := []struct{ path, want string }{
|
||||
{"tenant/pg", "tenant/pg/srv/base/20261021T100000/"},
|
||||
{"", "srv/base/20261021T100000/"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := barmanBaseBackupPrefix(tc.path, "srv", "20261021T100000"); got != tc.want {
|
||||
t.Errorf("barmanBaseBackupPrefix(%q) = %q, want %q", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBarmanS3DestinationPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, bucket, path string
|
||||
ok bool
|
||||
}{
|
||||
{"s3://bkt/tenant/pg/", "bkt", "tenant/pg", true},
|
||||
{"s3://bkt", "bkt", "", true},
|
||||
{"s3://bkt/", "bkt", "", true},
|
||||
{"gs://bkt/x/", "", "", false},
|
||||
{"bkt/x/", "", "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
bucket, path, err := parseBarmanS3DestinationPath(tc.in)
|
||||
if (err == nil) != tc.ok || bucket != tc.bucket || path != tc.path {
|
||||
t.Errorf("parseBarmanS3DestinationPath(%q) = %q, %q, %v", tc.in, bucket, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCNPGArtifactSizeBytes(t *testing.T) {
|
||||
const prefix = "tenant/pg/srv/base/20261021T100000/"
|
||||
|
||||
t.Run("sums every object of the base backup", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: prefix + "backup.info", Size: 1},
|
||||
{Key: prefix + "data_0000.tar.gz", Size: 1000},
|
||||
{Key: prefix + "data_0001.tar.gz", Size: 200},
|
||||
{Key: prefix + "16385.tar.gz", Size: 30},
|
||||
}}
|
||||
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"))
|
||||
got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000")
|
||||
if got != 1231 {
|
||||
t.Errorf("size = %d, want 1231", got)
|
||||
}
|
||||
if l.bucket != "bkt" || l.prefix != prefix || !l.recursive {
|
||||
t.Errorf("listed %q/%q recursive=%v, want bkt/%q recursive", l.bucket, l.prefix, l.recursive, prefix)
|
||||
}
|
||||
if l.conn.Endpoint != "s3.example:8443" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.SecretKey != "secret" {
|
||||
t.Errorf("connection = %+v", l.conn)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) {
|
||||
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
|
||||
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
withShortProbeTimeout(t, func() {
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000"); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("an empty endpointURL means AWS", func(t *testing.T) {
|
||||
store := newSizedBarmanObjectStore()
|
||||
store.EndpointURL = ""
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}}
|
||||
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"))
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 {
|
||||
t.Errorf("size = %d, want 5", got)
|
||||
}
|
||||
if l.conn.Endpoint != "s3.amazonaws.com" || !l.conn.Secure {
|
||||
t.Errorf("connection = %+v, want s3.amazonaws.com over TLS", l.conn)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("honours custom credential keys and the endpoint CA", func(t *testing.T) {
|
||||
store := newSizedBarmanObjectStore()
|
||||
store.S3Credentials.AccessKeyIDKey = "id"
|
||||
store.S3Credentials.SecretAccessKeyKey = "key"
|
||||
store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca"}}
|
||||
creds := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"},
|
||||
Data: map[string][]byte{"id": []byte("ID2"), "key": []byte("KEY2")},
|
||||
}
|
||||
ca := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"},
|
||||
Data: map[string][]byte{"ca.crt": []byte("PEM")},
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}}
|
||||
r := reconcilerWithLister(t, l, creds, ca)
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 {
|
||||
t.Errorf("size = %d, want 5", got)
|
||||
}
|
||||
if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" || !l.conn.CAOnly {
|
||||
t.Errorf("connection = %+v", l.conn)
|
||||
}
|
||||
})
|
||||
|
||||
unset := map[string]struct {
|
||||
store func() *strategyv1alpha1.BarmanObjectStoreTemplate
|
||||
backupID string
|
||||
lister *fakeS3Lister
|
||||
noSecret bool
|
||||
}{
|
||||
"unknown backup ID": {
|
||||
store: newSizedBarmanObjectStore, lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
|
||||
},
|
||||
"no objects under the prefix": {
|
||||
store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{},
|
||||
},
|
||||
"list error": {
|
||||
store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{err: errors.New("AccessDenied")},
|
||||
},
|
||||
"missing credentials Secret": {
|
||||
store: newSizedBarmanObjectStore, backupID: "id", noSecret: true,
|
||||
lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
|
||||
},
|
||||
"no s3Credentials": {
|
||||
store: func() *strategyv1alpha1.BarmanObjectStoreTemplate {
|
||||
s := newSizedBarmanObjectStore()
|
||||
s.S3Credentials = nil
|
||||
return s
|
||||
},
|
||||
backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
|
||||
},
|
||||
"non-s3 destination": {
|
||||
store: func() *strategyv1alpha1.BarmanObjectStoreTemplate {
|
||||
s := newSizedBarmanObjectStore()
|
||||
s.DestinationPath = "azure://container/x"
|
||||
return s
|
||||
},
|
||||
backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
|
||||
},
|
||||
"endpointURL with a path": {
|
||||
store: func() *strategyv1alpha1.BarmanObjectStoreTemplate {
|
||||
s := newSizedBarmanObjectStore()
|
||||
s.EndpointURL = "https://s3.example:8443/prefix"
|
||||
return s
|
||||
},
|
||||
backupID: "20261021T100000", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
|
||||
},
|
||||
}
|
||||
for name, tc := range unset {
|
||||
t.Run(name+" leaves size unset", func(t *testing.T) {
|
||||
var r *BackupJobReconciler
|
||||
if tc.noSecret {
|
||||
r = reconcilerWithLister(t, tc.lister)
|
||||
} else {
|
||||
r = reconcilerWithLister(t, tc.lister, s3CredsSecret("tenant", "creds"))
|
||||
}
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", tc.store(), "srv", tc.backupID); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCNPGArtifactSizeBytes_MinioClient drives the real minio client against
|
||||
// a stub ListObjectsV2 endpoint, so the endpoint URL parsing, Secret wiring
|
||||
// and response parsing are exercised together.
|
||||
func TestCNPGArtifactSizeBytes_MinioClient(t *testing.T) {
|
||||
const prefix = "tenant/pg/srv/base/20261021T100000/"
|
||||
var gotPath, gotPrefix, gotDelimiter, gotAuth string
|
||||
srv := httptest.NewServer(cnpgListStub(prefix, func(req *http.Request) {
|
||||
gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization")
|
||||
gotDelimiter = req.URL.Query().Get("delimiter")
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
|
||||
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
store := newSizedBarmanObjectStore()
|
||||
store.EndpointURL = srv.URL
|
||||
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 {
|
||||
t.Fatalf("size = %d, want 4103", got)
|
||||
}
|
||||
if gotPath != "/bkt/" || gotPrefix != prefix || gotDelimiter != "" {
|
||||
t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, %q and a recursive listing", gotPath, gotPrefix, gotDelimiter, prefix)
|
||||
}
|
||||
if !strings.Contains(gotAuth, "Credential=AKID/") {
|
||||
t.Errorf("request not signed with the Secret's access key: %q", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
if _, ok := req.URL.Query()["location"]; ok {
|
||||
fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`)
|
||||
return
|
||||
}
|
||||
if seen != nil {
|
||||
seen(req)
|
||||
}
|
||||
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>bkt</Name><Prefix>%[1]s</Prefix><KeyCount>2</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>%[1]sbackup.info</Key><Size>7</Size><ETag>"a"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
|
||||
<Contents><Key>%[1]sdata.tar.gz</Key><Size>4096</Size><ETag>"b"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`, prefix)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCNPGArtifactSizeBytes_EndpointCA serves the listing over TLS with a
|
||||
// self-signed certificate: the probe must trust it only through endpointCA.
|
||||
func TestCNPGArtifactSizeBytes_EndpointCA(t *testing.T) {
|
||||
const prefix = "tenant/pg/srv/base/20261021T100000/"
|
||||
srv := httptest.NewTLSServer(cnpgListStub(prefix, nil))
|
||||
defer srv.Close()
|
||||
caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
|
||||
|
||||
ca := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-ca"},
|
||||
Data: map[string][]byte{"ca.crt": caPEM},
|
||||
}
|
||||
c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"), ca)
|
||||
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
store := newSizedBarmanObjectStore()
|
||||
store.EndpointURL = srv.URL
|
||||
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 0 {
|
||||
t.Errorf("without endpointCA: size = %d, want 0 (untrusted certificate)", got)
|
||||
}
|
||||
store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "s3-ca"}}
|
||||
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 {
|
||||
t.Errorf("with endpointCA: size = %d, want 4103", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) {
|
||||
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
|
||||
strategyGroup := strategyv1alpha1.GroupVersion.Group
|
||||
j := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
|
||||
},
|
||||
}
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"},
|
||||
}
|
||||
cnpgBk := &cnpgtypes.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"},
|
||||
Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"},
|
||||
}
|
||||
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 777}}}
|
||||
r := reconcilerWithLister(t, l)
|
||||
r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
|
||||
|
||||
got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
|
||||
if err != nil {
|
||||
t.Fatalf("createCNPGBackupArtifact: %v", err)
|
||||
}
|
||||
if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 777 {
|
||||
t.Fatalf("Artifact = %#v, want SizeBytes 777", got.Status.Artifact)
|
||||
}
|
||||
if l.prefix != "tenant/pg/srv/base/20261021T100000/" {
|
||||
t.Errorf("listed prefix %q does not use the effective serverName", l.prefix)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) {
|
||||
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
|
||||
j := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
|
||||
},
|
||||
}
|
||||
existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}
|
||||
r := reconcilerWithLister(t, l)
|
||||
r.Client = newCNPGStrategyTestClient(t, existing, s3CredsSecret("tenant", "creds"))
|
||||
cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}}
|
||||
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()}
|
||||
|
||||
if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")); err != nil {
|
||||
t.Fatalf("createCNPGBackupArtifact: %v", err)
|
||||
}
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a Backup that already exists", l.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) {
|
||||
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
|
||||
j := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
|
||||
},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}}
|
||||
store := newCNPGStrategyTestClient(t, backup)
|
||||
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)}
|
||||
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}}
|
||||
got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
|
||||
if err != nil || got == nil || got.Name != "bj-race" {
|
||||
t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) {
|
||||
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
|
||||
j := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
|
||||
},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}}
|
||||
store := newCNPGStrategyTestClient(t, backup)
|
||||
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)}
|
||||
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}}
|
||||
_, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
|
||||
if !errors.Is(err, errBackupNotReadBack) {
|
||||
t.Errorf("err = %v, want errBackupNotReadBack", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileCNPG_UnreadableBackupIsRetried drives a completed backup
|
||||
// through the reconcile: when the Backup exists but cannot be read back,
|
||||
// the reconcile is retried and the BackupJob is not failed.
|
||||
func TestReconcileCNPG_UnreadableBackupIsRetried(t *testing.T) {
|
||||
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)),
|
||||
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
|
||||
newRolloutInstancePod(time.Now().Add(-time.Hour)))
|
||||
_, backups := f.pass()
|
||||
if len(backups) != 1 {
|
||||
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups))
|
||||
}
|
||||
ctx := context.Background()
|
||||
completed := backups[0].DeepCopy()
|
||||
completed.Status.Phase = cnpgBackupPhaseComplete
|
||||
completed.Status.BackupID = "20261021T100000"
|
||||
if err := f.c.Update(ctx, completed); err != nil {
|
||||
t.Fatalf("complete cnpg.io/Backup: %v", err)
|
||||
}
|
||||
if err := f.c.Create(ctx, &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "bj"}}); err != nil {
|
||||
t.Fatalf("seed Backup: %v", err)
|
||||
}
|
||||
f.r.APIReader = failingReads(t, f.c)
|
||||
|
||||
j := &backupsv1alpha1.BackupJob{}
|
||||
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); !errors.Is(err, errBackupNotReadBack) {
|
||||
t.Fatalf("reconcileCNPG err = %v, want errBackupNotReadBack so the reconcile is retried", err)
|
||||
}
|
||||
persisted := &backupsv1alpha1.BackupJob{}
|
||||
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Error("the BackupJob was failed although its Backup exists")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) {
|
||||
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
|
||||
j := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
|
||||
},
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}
|
||||
r := reconcilerWithLister(t, l)
|
||||
r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
|
||||
r.APIReader = failingReads(t, newCNPGStrategyTestClient(t))
|
||||
cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}}
|
||||
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()}
|
||||
|
||||
got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("createCNPGBackupArtifact = %v, %v; a transient read must not fail the backup", got, err)
|
||||
}
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 probed %d times after the existence check failed", l.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileCNPG_RejectedCreateFailsTheJob pins that only the read-back
|
||||
// is retried: a Create the apiserver rejects still fails the BackupJob.
|
||||
func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) {
|
||||
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)),
|
||||
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
|
||||
newRolloutInstancePod(time.Now().Add(-time.Hour)))
|
||||
_, backups := f.pass()
|
||||
if len(backups) != 1 {
|
||||
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups))
|
||||
}
|
||||
ctx := context.Background()
|
||||
completed := backups[0].DeepCopy()
|
||||
completed.Status.Phase = cnpgBackupPhaseComplete
|
||||
if err := f.c.Update(ctx, completed); err != nil {
|
||||
t.Fatalf("complete cnpg.io/Backup: %v", err)
|
||||
}
|
||||
f.r.Client = forbidBackupCreate(t, f.c)
|
||||
|
||||
j := &backupsv1alpha1.BackupJob{}
|
||||
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil {
|
||||
t.Fatalf("reconcileCNPG err = %v, want the BackupJob failed rather than retried", err)
|
||||
}
|
||||
persisted := &backupsv1alpha1.BackupJob{}
|
||||
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase)
|
||||
}
|
||||
}
|
||||
@@ -435,7 +435,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
}
|
||||
artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app)
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
|
||||
return r.failBackupArtifact(ctx, j, err)
|
||||
}
|
||||
now := metav1.Now()
|
||||
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
|
||||
@@ -815,6 +815,10 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
|
||||
rendered *strategyv1alpha1.CNPGTemplate,
|
||||
sourceApp *postgresapp.Postgres,
|
||||
) (*backupsv1alpha1.Backup, error) {
|
||||
existing, probe := r.existingBackup(ctx, j.Namespace, j.Name)
|
||||
if existing != nil {
|
||||
return existing, nil
|
||||
}
|
||||
takenAt := metav1.Now()
|
||||
if cnpgBackup.Status.StartedAt != nil && !cnpgBackup.Status.StartedAt.IsZero() {
|
||||
takenAt = *cnpgBackup.Status.StartedAt
|
||||
@@ -860,6 +864,9 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
|
||||
UnderlyingResources: underlyingResources,
|
||||
},
|
||||
}
|
||||
if probe {
|
||||
backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID)
|
||||
}
|
||||
if j.Spec.PlanRef != nil {
|
||||
backup.Spec.PlanRef = j.Spec.PlanRef
|
||||
}
|
||||
@@ -872,8 +879,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
|
||||
return nil, err
|
||||
}
|
||||
existing := &backupsv1alpha1.Backup{}
|
||||
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
|
||||
return nil, getErr
|
||||
if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
|
||||
return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr)
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
// Slack around the [created, completed] window of the operator Backup when
|
||||
// matching a dump: the timestamp in its file name comes from the backup Job
|
||||
// pod's clock and is truncated to seconds, and its LastModified comes from
|
||||
// the object store's clock, neither of which is the apiserver's.
|
||||
const mariadbSizeProbeClockSkew = time.Minute
|
||||
|
||||
var (
|
||||
errMariaDBDumpNotFound = errors.New("no dump in the bucket belongs to the operator Backup")
|
||||
errMariaDBDumpAmbiguous = errors.New("the dump of the operator Backup cannot be told apart from another run's")
|
||||
)
|
||||
|
||||
// mariadbDump is a logical dump mariadb-operator wrote to the bucket.
|
||||
type mariadbDump struct {
|
||||
// Key is the object key; Name is the file name, Key without the prefix.
|
||||
Key string
|
||||
Name string
|
||||
// Timestamp is the one in the file name, taken by the backup Job pod's
|
||||
// clock as the dump started.
|
||||
Timestamp time.Time
|
||||
Size int64
|
||||
// LastModified is the object store's clock as the upload finished.
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for
|
||||
// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump
|
||||
// cannot be found or told apart: an unknown size must not be reported as a
|
||||
// measured one.
|
||||
func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 {
|
||||
ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout)
|
||||
defer cancel()
|
||||
dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt)
|
||||
if err != nil {
|
||||
getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err)
|
||||
return 0
|
||||
}
|
||||
return dump.Size
|
||||
}
|
||||
|
||||
// findMariaDBDump lists the bucket mdbBackup wrote to and returns the one
|
||||
// dump that belongs to it, as matchMariaDBDump decides. The location and
|
||||
// credentials come from mdbBackup's own spec: the strategy it was rendered
|
||||
// from can have been edited since, and pointing at another bucket now says
|
||||
// nothing about where this dump went.
|
||||
func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) {
|
||||
s3 := mdbBackup.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return mariadbDump{}, errors.New("the operator Backup does not store to S3")
|
||||
}
|
||||
others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err)
|
||||
}
|
||||
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err)
|
||||
}
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("build S3 client: %w", err)
|
||||
}
|
||||
// Non-recursive, as mariadb-operator lists it: the default prefix is also
|
||||
// the default CNPG destinationPath, so an app of each kind with the same
|
||||
// name shares it and a recursive listing would walk the WAL archive.
|
||||
prefix := mariadbOperatorS3Prefix(s3.Prefix)
|
||||
objects, err := lister.ListObjects(ctx, s3.Bucket, prefix, false)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, prefix, err)
|
||||
}
|
||||
own := newMariaDBRun(mdbBackup, mdbBackup.CreationTimestamp.Time, completedAt)
|
||||
return matchMariaDBDump(objects, prefix, own, others)
|
||||
}
|
||||
|
||||
// mariadbRun is the span in which an operator Backup can have written its
|
||||
// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side.
|
||||
type mariadbRun struct {
|
||||
name string
|
||||
lower, upper time.Time
|
||||
}
|
||||
|
||||
func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun {
|
||||
return mariadbRun{
|
||||
name: b.Namespace + "/" + b.Name,
|
||||
lower: created.Add(-mariadbSizeProbeClockSkew),
|
||||
upper: completed.Add(mariadbSizeProbeClockSkew),
|
||||
}
|
||||
}
|
||||
|
||||
func (r mariadbRun) placesOnBothClocks(d mariadbDump) bool {
|
||||
return inWindow(d.Timestamp, r.lower, r.upper) && inWindow(d.LastModified, r.lower, r.upper)
|
||||
}
|
||||
|
||||
func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool {
|
||||
return inWindow(d.Timestamp, r.lower, r.upper) || inWindow(d.LastModified, r.lower, r.upper)
|
||||
}
|
||||
|
||||
// matchMariaDBDump picks the dump of run own among objects, which share a
|
||||
// prefix with every other run of the application. A dump carries two
|
||||
// clocks, the backup pod's in its file name and the object store's in its
|
||||
// LastModified, and either can be skewed. A dump is taken only when it is
|
||||
// the one dump either clock places in own's window, both clocks place it
|
||||
// there, and neither places it in the window of another finished run: a
|
||||
// dump in the window on one clock only may be this run's own, misplaced by
|
||||
// a skewed clock, which leaves any other candidate unproven. A neighbour's
|
||||
// dump is still taken if this run's own is missing from the listing, or
|
||||
// misplaced on both clocks, and the neighbour's operator Backup is gone or
|
||||
// has not completed.
|
||||
func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) {
|
||||
var near []mariadbDump
|
||||
for _, obj := range objects {
|
||||
name := strings.TrimPrefix(obj.Key, prefix)
|
||||
ts, ok := parseMariaDBLogicalBackupFile(name)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
d := mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified}
|
||||
if own.placesOnEitherClock(d) {
|
||||
near = append(near, d)
|
||||
}
|
||||
}
|
||||
if len(near) == 0 {
|
||||
return mariadbDump{}, errMariaDBDumpNotFound
|
||||
}
|
||||
if len(near) > 1 {
|
||||
return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime on at least one clock", errMariaDBDumpAmbiguous, len(near))
|
||||
}
|
||||
d := near[0]
|
||||
if !own.placesOnBothClocks(d) {
|
||||
return mariadbDump{}, fmt.Errorf("%w: %s falls in its lifetime on one clock only", errMariaDBDumpAmbiguous, d.Name)
|
||||
}
|
||||
for _, other := range others {
|
||||
if other.placesOnEitherClock(d) {
|
||||
return mariadbDump{}, fmt.Errorf("%w: %s also falls in the lifetime of operator Backup %s", errMariaDBDumpAmbiguous, d.Name, other.name)
|
||||
}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// finishedMariaDBRuns returns the runs of the other operator Backups that
|
||||
// write under the same bucket and prefix as b and have completed, whether
|
||||
// their Job succeeded or failed. One still running is left out: its window
|
||||
// has no end yet, and an operator Backup stuck short of Complete, as one
|
||||
// whose Job pod never starts is, would otherwise contest every later run of
|
||||
// the application for good. The endpoint is not compared, since one has
|
||||
// several spellings, and a false neighbour only leaves a size unset.
|
||||
func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) {
|
||||
list := &mariadbtypes.BackupList{}
|
||||
if err := reader.List(ctx, list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var runs []mariadbRun
|
||||
for i := range list.Items {
|
||||
other := &list.Items[i]
|
||||
if other.Namespace == b.Namespace && other.Name == b.Name {
|
||||
continue
|
||||
}
|
||||
s3 := other.Spec.Storage.S3
|
||||
if s3 == nil || s3.Bucket != b.Spec.Storage.S3.Bucket || mariadbOperatorS3Prefix(s3.Prefix) != mariadbOperatorS3Prefix(b.Spec.Storage.S3.Prefix) {
|
||||
continue
|
||||
}
|
||||
cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete)
|
||||
if cond == nil || cond.Status != metav1.ConditionTrue {
|
||||
continue
|
||||
}
|
||||
runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time))
|
||||
}
|
||||
return runs, nil
|
||||
}
|
||||
|
||||
// mariadbS3Connection resolves the Secrets and TLS settings of the operator
|
||||
// Backup's storage, so the bucket is read with the tenant's credentials
|
||||
// rather than the controller's.
|
||||
func mariadbS3Connection(ctx context.Context, store objectStore, namespace string, s3 *mariadbtypes.S3Storage) (s3Connection, error) {
|
||||
conn := s3Connection{
|
||||
Endpoint: s3.Endpoint,
|
||||
Region: s3.Region,
|
||||
Secure: s3.TLS != nil && s3.TLS.Enabled,
|
||||
}
|
||||
var err error
|
||||
if conn.AccessKey, err = store.secretValue(ctx, namespace, s3.AccessKeyIdSecretKeyRef.Name, s3.AccessKeyIdSecretKeyRef.Key); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if conn.SecretKey, err = store.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if ref := s3.SessionTokenSecretKeyRef; ref != nil {
|
||||
if conn.SessionToken, err = store.secretValue(ctx, namespace, ref.Name, ref.Key); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
}
|
||||
if conn.Secure && s3.TLS.CASecretKeyRef != nil {
|
||||
ca, err := store.secretValue(ctx, namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
conn.CAPEM = []byte(ca)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// mariadbOperatorS3Prefix mirrors how mariadb-operator normalises
|
||||
// spec.storage.s3.prefix before prepending it to object keys.
|
||||
func mariadbOperatorS3Prefix(prefix string) string {
|
||||
if prefix == "" || prefix == "/" {
|
||||
return ""
|
||||
}
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
return prefix + "/"
|
||||
}
|
||||
return prefix
|
||||
}
|
||||
|
||||
func inWindow(t, lower, upper time.Time) bool {
|
||||
return !t.Before(lower) && !t.After(upper)
|
||||
}
|
||||
|
||||
// Compression tokens mariadb-operator puts in a dump file name: the
|
||||
// algorithm before .sql in v25.10 (backup.<ts>.gzip.sql), the extension
|
||||
// after it in later releases (backup.<ts>.sql.gz).
|
||||
var (
|
||||
mariadbDumpAlgorithms = map[string]bool{"none": true, "bzip2": true, "gzip": true, "zstd": true}
|
||||
mariadbDumpExtensions = map[string]bool{"bz2": true, "gz": true, "zst": true}
|
||||
)
|
||||
|
||||
// parseMariaDBLogicalBackupFile accepts the logical-backup file names
|
||||
// mariadb-operator writes and returns the embedded timestamp.
|
||||
func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) {
|
||||
parts := strings.Split(name, ".")
|
||||
if parts[0] != "backup" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
switch {
|
||||
case len(parts) == 3 && parts[2] == "sql":
|
||||
case len(parts) == 4 && parts[2] == "sql" && mariadbDumpExtensions[parts[3]]:
|
||||
case len(parts) == 4 && parts[3] == "sql" && mariadbDumpAlgorithms[parts[2]]:
|
||||
default:
|
||||
return time.Time{}, false
|
||||
}
|
||||
ts, err := time.Parse(time.RFC3339, parts[1])
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return ts, true
|
||||
}
|
||||
@@ -0,0 +1,750 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
func dumpKey(prefix string, ts time.Time, compression string) string {
|
||||
if compression == "" {
|
||||
return fmt.Sprintf("%sbackup.%s.sql", prefix, ts.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression)
|
||||
}
|
||||
|
||||
// mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a
|
||||
// Backup whose Job succeeded. A failed Job also gets Status=True, with
|
||||
// Reason=JobFailed, so a fixture without the reason is not a success.
|
||||
func mariadbBackupSucceeded(at metav1.Time) metav1.Condition {
|
||||
return metav1.Condition{
|
||||
Type: mariadbtypes.ConditionTypeComplete,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: mariadbtypes.ConditionReasonJobComplete,
|
||||
Message: "Success",
|
||||
LastTransitionTime: at,
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMariaDBLogicalBackupFile(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"backup.2026-10-21T10:00:00Z.sql": true,
|
||||
"backup.2026-10-21T10:00:00Z.gzip.sql": true,
|
||||
"backup.2026-10-21T10:00:00+04:00.sql": true,
|
||||
"backup.2026-10-21T10:00:00Z.sql.tmp": false,
|
||||
"backup.2026-10-21T10:00:00Z.sql.gz": true,
|
||||
"backup.2026-10-21T10:00:00Z.sql.zst": true,
|
||||
"backup.2026-10-21T10:00:00Z.zstd.sql": true,
|
||||
"backup.2026-10-21T10:00:00Z.xz.sql": false,
|
||||
"backup.2026-10-21T10:00:00Z.sql.gz.tmp": false,
|
||||
"physicalbackup-20261021100000.xb": false,
|
||||
"backup.not-a-date.sql": false,
|
||||
"sub/backup.2026-10-21T10:00:00Z.sql": false,
|
||||
"backup.2026-10-21T10:00:00Z.a.gzip.sql": false,
|
||||
"backups.2026-10-21T10:00:00Z.gzip.sql": false,
|
||||
"backup.2026-10-21T10:00:00Z.gzip.dump": false,
|
||||
"": false,
|
||||
}
|
||||
for name, want := range cases {
|
||||
if _, got := parseMariaDBLogicalBackupFile(name); got != want {
|
||||
t.Errorf("parseMariaDBLogicalBackupFile(%q) = %v, want %v", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMariaDBOperatorS3Prefix(t *testing.T) {
|
||||
cases := map[string]string{"": "", "/": "", "ns/app": "ns/app/", "ns/app/": "ns/app/"}
|
||||
for in, want := range cases {
|
||||
if got := mariadbOperatorS3Prefix(in); got != want {
|
||||
t.Errorf("mariadbOperatorS3Prefix(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// operatorBackupStoringTo returns the operator Backup a strategy with s3
|
||||
// storage renders, created at created in namespace tenant.
|
||||
func operatorBackupStoringTo(t *testing.T, s3 *strategyv1alpha1.MariaDBS3Template, created time.Time) *mariadbtypes.Backup {
|
||||
t.Helper()
|
||||
storage, err := buildMariaDBBackupStorage(strategyv1alpha1.MariaDBStorageTemplate{S3: s3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &mariadbtypes.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created)},
|
||||
Spec: mariadbtypes.BackupSpec{Storage: storage},
|
||||
}
|
||||
}
|
||||
|
||||
func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
completed := created.Add(3 * time.Minute)
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Prefix = "tenant/db"
|
||||
mdbBackup := operatorBackupStoringTo(t, s3, created)
|
||||
const prefix = "tenant/db/"
|
||||
|
||||
t.Run("single dump inside the backup window", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111, LastModified: created.Add(-24 * time.Hour)},
|
||||
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)},
|
||||
{Key: prefix + "0-backup-target.txt", Size: 7},
|
||||
}}
|
||||
got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed)
|
||||
if got != 4242 {
|
||||
t.Errorf("size = %d, want 4242", got)
|
||||
}
|
||||
if l.conn.AccessKey != "AKID" || l.conn.Endpoint != "s3.example:9000" {
|
||||
t.Errorf("connection = %+v, want the Secret's key and the rendered endpoint", l.conn)
|
||||
}
|
||||
if l.bucket != "bkt" || l.prefix != prefix || l.recursive {
|
||||
t.Errorf("listed %q/%q recursive=%v, want bkt/%q non-recursive", l.bucket, l.prefix, l.recursive, prefix)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reads the endpoint CA from its Secret", func(t *testing.T) {
|
||||
tlsS3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
tlsS3.Prefix = "tenant/db"
|
||||
tlsS3.TLS = &strategyv1alpha1.MariaDBS3TLS{
|
||||
Enabled: true,
|
||||
CASecretKeyRef: &strategyv1alpha1.MariaDBSecretKeySelector{Name: "ca", Key: "ca.crt"},
|
||||
}
|
||||
ca := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"},
|
||||
Data: map[string][]byte{"ca.crt": []byte("PEM")},
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}}
|
||||
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca)
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 {
|
||||
t.Errorf("size = %d, want 3", got)
|
||||
}
|
||||
if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" {
|
||||
t.Errorf("connection = %+v, want TLS with the Secret's CA", l.conn)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reads the session token from its Secret", func(t *testing.T) {
|
||||
stsS3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
stsS3.Prefix = "tenant/db"
|
||||
stsS3.SessionTokenSecretKeyRef = &strategyv1alpha1.MariaDBSecretKeySelector{Name: "sts", Key: "token"}
|
||||
sts := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "sts"},
|
||||
Data: map[string][]byte{"token": []byte("TOKEN")},
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}}
|
||||
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts)
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 {
|
||||
t.Errorf("size = %d, want 4", got)
|
||||
}
|
||||
if l.conn.SessionToken != "TOKEN" {
|
||||
t.Errorf("session token = %q, want the Secret's", l.conn.SessionToken)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pod clock skew within tolerance", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 {
|
||||
t.Errorf("size = %d, want 9", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a skewed pod clock does not attribute a neighbouring run's dump", func(t *testing.T) {
|
||||
// The pod clock runs 5m behind. This run's dump is named outside the
|
||||
// window; the previous run's dump, uploaded before this run started,
|
||||
// is named inside it.
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)},
|
||||
{Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a dump uploaded just after Complete is inside the slack", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 {
|
||||
t.Errorf("size = %d, want 5", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) {
|
||||
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
|
||||
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
withShortProbeTimeout(t, func() {
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("two dumps in the window leave size unset", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)},
|
||||
{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
t.Errorf("size = %d, want 0 (ambiguous)", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no dump in the window leaves size unset", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("list error leaves size unset", func(t *testing.T) {
|
||||
l := &fakeS3Lister{err: errors.New("AccessDenied")}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing credentials Secret leaves size unset", func(t *testing.T) {
|
||||
c := newMariaDBStrategyTestClient(t)
|
||||
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestMatchMariaDBDump pins what a restore reads off a match: the object
|
||||
// itself, and errors it can tell apart.
|
||||
func TestMatchMariaDBDump(t *testing.T) {
|
||||
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
completed := created.Add(3 * time.Minute)
|
||||
const prefix = "tenant/db/"
|
||||
own := s3ObjectInfo{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(2 * time.Minute)}
|
||||
run := newMariaDBRun(&mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, created, completed)
|
||||
|
||||
got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, run, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("matchMariaDBDump: %v", err)
|
||||
}
|
||||
want := mariadbDump{
|
||||
Key: own.Key,
|
||||
Name: strings.TrimPrefix(own.Key, prefix),
|
||||
Timestamp: created.Add(time.Minute),
|
||||
Size: 4242,
|
||||
LastModified: own.LastModified,
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("matchMariaDBDump = %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
if _, err := matchMariaDBDump(nil, prefix, run, nil); !errors.Is(err, errMariaDBDumpNotFound) {
|
||||
t.Errorf("no dump: err = %v, want errMariaDBDumpNotFound", err)
|
||||
}
|
||||
second := s3ObjectInfo{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 1, LastModified: created.Add(2 * time.Minute)}
|
||||
if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, run, nil); !errors.Is(err, errMariaDBDumpAmbiguous) {
|
||||
t.Errorf("two dumps: err = %v, want errMariaDBDumpAmbiguous", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindMariaDBDump pins that a dump is read with nothing but a client,
|
||||
// as a reconciler other than BackupJobReconciler has, and that an operator
|
||||
// Backup that does not store to S3 is refused without a listing.
|
||||
func TestFindMariaDBDump(t *testing.T) {
|
||||
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
completed := created.Add(3 * time.Minute)
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Prefix = "tenant/db"
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey("tenant/db/", created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)},
|
||||
}}
|
||||
store := objectStore{
|
||||
reader: newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")),
|
||||
newLister: func(conn s3Connection) (s3ObjectLister, error) {
|
||||
l.conn = conn
|
||||
return l, nil
|
||||
},
|
||||
}
|
||||
|
||||
got, err := findMariaDBDump(context.Background(), store, operatorBackupStoringTo(t, s3, created), completed)
|
||||
if err != nil || got.Size != 4242 {
|
||||
t.Fatalf("findMariaDBDump = %+v, %v; want the 4242-byte dump", got, err)
|
||||
}
|
||||
if l.conn.AccessKey != "AKID" {
|
||||
t.Errorf("connection = %+v, want the Secret's key", l.conn)
|
||||
}
|
||||
|
||||
pvc := &mariadbtypes.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-pvc", CreationTimestamp: metav1.NewTime(created)},
|
||||
Spec: mariadbtypes.BackupSpec{Storage: mariadbtypes.BackupStorage{PersistentVolumeClaim: &corev1.PersistentVolumeClaimSpec{}}},
|
||||
}
|
||||
if _, err := findMariaDBDump(context.Background(), store, pvc, completed); err == nil {
|
||||
t.Error("findMariaDBDump on a PVC-backed operator Backup returned no error")
|
||||
}
|
||||
if l.calls != 1 {
|
||||
t.Errorf("S3 listed %d times, want once: a PVC-backed Backup has nothing to list", l.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindMariaDBDump_Ownership drives runs of one application that share
|
||||
// a prefix. This run lives from t0 to t0+40s, so its window with the
|
||||
// one-minute slack is [t0-1m, t0+1m40s]; its own dump is 123 bytes and a
|
||||
// neighbour's is 666 or 777. A neighbour's dump must never be returned.
|
||||
func TestFindMariaDBDump_Ownership(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Prefix = "tenant/db"
|
||||
const prefix = "tenant/db/"
|
||||
run := func(name string, created, completed time.Time) *mariadbtypes.Backup {
|
||||
b := operatorBackupStoringTo(t, s3, created)
|
||||
b.Name = name
|
||||
b.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(completed))}
|
||||
return b
|
||||
}
|
||||
dump := func(named, uploaded time.Time, size int64) s3ObjectInfo {
|
||||
return s3ObjectInfo{Key: dumpKey(prefix, named, "gzip"), Size: size, LastModified: uploaded}
|
||||
}
|
||||
own := run("op-bk", t0, t0.Add(40*time.Second))
|
||||
find := func(t *testing.T, objects []s3ObjectInfo, operatorBackups ...client.Object) (mariadbDump, error) {
|
||||
t.Helper()
|
||||
store := objectStore{
|
||||
reader: newMariaDBStrategyTestClient(t, append(operatorBackups, own, s3CredsSecret("tenant", "creds"))...),
|
||||
newLister: func(s3Connection) (s3ObjectLister, error) {
|
||||
return &fakeS3Lister{objects: objects}, nil
|
||||
},
|
||||
}
|
||||
return findMariaDBDump(context.Background(), store, own, t0.Add(40*time.Second))
|
||||
}
|
||||
wantAmbiguous := func(t *testing.T, got mariadbDump, err error) {
|
||||
t.Helper()
|
||||
if !errors.Is(err, errMariaDBDumpAmbiguous) {
|
||||
t.Errorf("findMariaDBDump = %+v, %v; want errMariaDBDumpAmbiguous", got, err)
|
||||
}
|
||||
}
|
||||
wantOwn := func(t *testing.T, got mariadbDump, err error) {
|
||||
t.Helper()
|
||||
if err != nil || got.Size != 123 {
|
||||
t.Errorf("findMariaDBDump = %+v, %v; want this run's 123-byte dump", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("a pod clock behind does not hand this run the previous run's dump", func(t *testing.T) {
|
||||
// The previous run finished 20s before this one was created and its
|
||||
// operator Backup is gone. This run's pod clock is 5m behind, so its
|
||||
// own dump is named outside the window and uploaded inside it.
|
||||
got, err := find(t, []s3ObjectInfo{
|
||||
dump(t0.Add(-30*time.Second), t0.Add(-20*time.Second), 666),
|
||||
dump(t0.Add(5*time.Second-5*time.Minute), t0.Add(30*time.Second), 123),
|
||||
})
|
||||
wantAmbiguous(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("a pod clock ahead does not hand this run the next run's dump", func(t *testing.T) {
|
||||
// The next run started 5s after this one completed and has uploaded
|
||||
// its dump by the time this one is probed; it has not completed yet.
|
||||
next := operatorBackupStoringTo(t, s3, t0.Add(45*time.Second))
|
||||
next.Name = "op-next"
|
||||
next.Status.Conditions = []metav1.Condition{{
|
||||
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
|
||||
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(45 * time.Second)),
|
||||
}}
|
||||
got, err := find(t, []s3ObjectInfo{
|
||||
dump(t0.Add(5*time.Second+5*time.Minute), t0.Add(30*time.Second), 123),
|
||||
dump(t0.Add(50*time.Second), t0.Add(80*time.Second), 777),
|
||||
}, next)
|
||||
wantAmbiguous(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("an object store clock behind does not hand this run the next run's dump", func(t *testing.T) {
|
||||
// The store clock is 2m behind: this run's upload is stamped before
|
||||
// the window opens, the next run's is pulled back into it.
|
||||
got, err := find(t, []s3ObjectInfo{
|
||||
dump(t0.Add(5*time.Second), t0.Add(30*time.Second-2*time.Minute), 123),
|
||||
dump(t0.Add(50*time.Second), t0.Add(3*time.Minute-2*time.Minute), 777),
|
||||
})
|
||||
wantAmbiguous(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("a dump another finished run wrote is not taken when this run's own is gone", func(t *testing.T) {
|
||||
// This run's dump was pruned; the next run's falls in this run's
|
||||
// window on both clocks, and in its own.
|
||||
next := run("op-next", t0.Add(45*time.Second), t0.Add(85*time.Second))
|
||||
got, err := find(t, []s3ObjectInfo{
|
||||
dump(t0.Add(50*time.Second), t0.Add(75*time.Second), 777),
|
||||
}, next)
|
||||
wantAmbiguous(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("a dump the window holds on one clock only is not taken", func(t *testing.T) {
|
||||
// This run's dump is gone; the previous run's operator Backup is too,
|
||||
// and its dump, started 2m before this run, finished uploading 20s
|
||||
// before it.
|
||||
got, err := find(t, []s3ObjectInfo{
|
||||
dump(t0.Add(-2*time.Minute), t0.Add(-20*time.Second), 666),
|
||||
})
|
||||
if err == nil {
|
||||
t.Errorf("findMariaDBDump = %+v; want no dump attributed", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a run still in progress does not contest this run's dump", func(t *testing.T) {
|
||||
// A concurrent run created 30s before this one has not uploaded yet.
|
||||
running := operatorBackupStoringTo(t, s3, t0.Add(-30*time.Second))
|
||||
running.Name = "op-running"
|
||||
running.Status.Conditions = []metav1.Condition{{
|
||||
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
|
||||
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-30 * time.Second)),
|
||||
}}
|
||||
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, running)
|
||||
wantOwn(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("runs minutes apart each keep their own dump", func(t *testing.T) {
|
||||
previous := run("op-previous", t0.Add(-4*time.Minute), t0.Add(-3*time.Minute))
|
||||
next := run("op-next", t0.Add(3*time.Minute), t0.Add(4*time.Minute))
|
||||
got, err := find(t, []s3ObjectInfo{
|
||||
dump(t0.Add(-230*time.Second), t0.Add(-190*time.Second), 666),
|
||||
dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123),
|
||||
dump(t0.Add(185*time.Second), t0.Add(220*time.Second), 777),
|
||||
}, previous, next)
|
||||
wantOwn(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("a run that writes elsewhere does not contest this run's dump", func(t *testing.T) {
|
||||
other := newRenderedMariaDBTemplate().Storage.S3
|
||||
other.Prefix = "tenant/other"
|
||||
elsewhere := operatorBackupStoringTo(t, other, t0)
|
||||
elsewhere.Name = "op-elsewhere"
|
||||
elsewhere.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(t0.Add(40 * time.Second)))}
|
||||
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, elsewhere)
|
||||
wantOwn(t, got, err)
|
||||
})
|
||||
|
||||
t.Run("a run stuck short of Complete does not contest later runs", func(t *testing.T) {
|
||||
// A backup Job whose pod never starts leaves its operator Backup
|
||||
// JobRunning for good.
|
||||
stuck := operatorBackupStoringTo(t, s3, t0.Add(-3*time.Hour))
|
||||
stuck.Name = "op-stuck"
|
||||
stuck.Status.Conditions = []metav1.Condition{{
|
||||
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
|
||||
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-3 * time.Hour)),
|
||||
}}
|
||||
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, stuck)
|
||||
wantOwn(t, got, err)
|
||||
})
|
||||
}
|
||||
|
||||
// TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client
|
||||
// against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and
|
||||
// response parsing are exercised together.
|
||||
func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) {
|
||||
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
key := dumpKey("tenant/db/", created.Add(time.Minute), "gzip")
|
||||
|
||||
var gotPath, gotPrefix, gotDelimiter, gotAuth string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization")
|
||||
gotDelimiter = req.URL.Query().Get("delimiter")
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>bkt</Name><Prefix>tenant/db/</Prefix><KeyCount>1</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>%s</Key><Size>31337</Size><ETag>"x"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
|
||||
<CommonPrefixes><Prefix>tenant/db/srv/</Prefix></CommonPrefixes>
|
||||
</ListBucketResult>`, key)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
|
||||
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Endpoint = strings.TrimPrefix(srv.URL, "http://")
|
||||
s3.Prefix = "tenant/db/"
|
||||
s3.Region = "us-east-1"
|
||||
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 {
|
||||
t.Fatalf("size = %d, want 31337", got)
|
||||
}
|
||||
if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" {
|
||||
t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, tenant/db/ and a non-recursive listing", gotPath, gotPrefix, gotDelimiter)
|
||||
}
|
||||
if !strings.Contains(gotAuth, "Credential=AKID/") {
|
||||
t.Errorf("request not signed with the Secret's access key: %q", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC))
|
||||
rendered := newRenderedMariaDBTemplate()
|
||||
rendered.Storage.S3.Prefix = "src/"
|
||||
mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, completed.Add(-3*time.Minute))
|
||||
mdbBackup.Name = "src-op-bk"
|
||||
mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)}
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
|
||||
Parameters: map[string]string{},
|
||||
}
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
r := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)},
|
||||
}}, job, s3CredsSecret("tenant", "creds"))
|
||||
|
||||
artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered)
|
||||
if err != nil {
|
||||
t.Fatalf("createMariaDBBackupArtifact: %v", err)
|
||||
}
|
||||
if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 {
|
||||
t.Fatalf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the
|
||||
// probe reads where the operator Backup wrote its dump, not where the
|
||||
// strategy, edited while the backup ran, now points.
|
||||
func TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC))
|
||||
original := newRenderedMariaDBTemplate().Storage.S3
|
||||
original.Prefix = "original/"
|
||||
mdbBackup := operatorBackupStoringTo(t, original, completed.Add(-3*time.Minute))
|
||||
mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)}
|
||||
edited := newRenderedMariaDBTemplate()
|
||||
edited.Storage.S3.Bucket = "replacement-bkt"
|
||||
edited.Storage.S3.Endpoint = "replacement.example:9000"
|
||||
edited.Storage.S3.Prefix = "replacement/"
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-edited"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey("original/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)},
|
||||
}}
|
||||
r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds"))
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
|
||||
Parameters: map[string]string{},
|
||||
}
|
||||
|
||||
artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, edited)
|
||||
if err != nil {
|
||||
t.Fatalf("createMariaDBBackupArtifact: %v", err)
|
||||
}
|
||||
if l.bucket != "bkt" || l.prefix != "original/" || l.conn.Endpoint != "s3.example:9000" {
|
||||
t.Errorf("listed %s/%s at %s, want bkt/original/ at s3.example:9000, where the operator Backup wrote", l.bucket, l.prefix, l.conn.Endpoint)
|
||||
}
|
||||
if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 {
|
||||
t.Errorf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
existing := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"},
|
||||
Status: backupsv1alpha1.BackupStatus{Artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bkt/src/op-bk", SizeBytes: 99}},
|
||||
}
|
||||
l := &fakeS3Lister{}
|
||||
r := reconcilerWithLister(t, l, job, existing, s3CredsSecret("tenant", "creds"))
|
||||
rendered := newRenderedMariaDBTemplate()
|
||||
rendered.Storage.S3.Prefix = "src/"
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
|
||||
Parameters: map[string]string{},
|
||||
}
|
||||
|
||||
got, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, rendered)
|
||||
if err != nil {
|
||||
t.Fatalf("createMariaDBBackupArtifact: %v", err)
|
||||
}
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a Backup that already exists", l.calls)
|
||||
}
|
||||
if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 99 {
|
||||
t.Errorf("Artifact = %#v, want the existing one", got.Status.Artifact)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
l := &fakeS3Lister{}
|
||||
r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds"))
|
||||
r.APIReader = failingReads(t, newMariaDBStrategyTestClient(t))
|
||||
got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, newRenderedMariaDBTemplate())
|
||||
if err != nil || got == nil || got.Name != "bj-transient" {
|
||||
t.Fatalf("createMariaDBBackupArtifact = %v, %v; a transient read must not fail the backup", got, err)
|
||||
}
|
||||
if err := r.Get(context.Background(), client.ObjectKeyFromObject(got), &backupsv1alpha1.Backup{}); err != nil {
|
||||
t.Errorf("the Backup was not created: %v", err)
|
||||
}
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 probed %d times after the existence check failed", l.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}}
|
||||
store := newMariaDBStrategyTestClient(t, job, backup)
|
||||
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store), Scheme: store.Scheme()}
|
||||
got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{})
|
||||
if err != nil || got == nil || got.Name != "bj-race" {
|
||||
t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMariaDBBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}}
|
||||
store := newMariaDBStrategyTestClient(t, job, backup)
|
||||
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store), Scheme: store.Scheme()}
|
||||
_, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{})
|
||||
if !errors.Is(err, errBackupNotReadBack) {
|
||||
t.Errorf("err = %v, want errBackupNotReadBack", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileMariaDB_UnreadableBackupIsRetried drives a completed backup
|
||||
// through the reconcile: when the Backup exists but cannot be read back,
|
||||
// the reconcile is retried and the BackupJob is not failed.
|
||||
func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) {
|
||||
strategyGroup := strategyv1alpha1.GroupVersion.Group
|
||||
now := metav1.Now()
|
||||
bj := newMariaDBBackupJob("bj-unreadable", "tenant")
|
||||
bj.Status.StartedAt = &now
|
||||
bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
|
||||
strategy := &strategyv1alpha1.MariaDB{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"},
|
||||
Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()},
|
||||
}
|
||||
operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}}
|
||||
operatorBackup := &mariadbtypes.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "op-bk",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: bj.Name,
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace,
|
||||
},
|
||||
},
|
||||
Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: bj.Name}}
|
||||
store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup, backup)
|
||||
r := &BackupJobReconciler{
|
||||
Client: store,
|
||||
APIReader: failingReads(t, store),
|
||||
Scheme: store.Scheme(),
|
||||
Recorder: record.NewFakeRecorder(10),
|
||||
}
|
||||
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name,
|
||||
}}
|
||||
|
||||
_, err := r.reconcileMariaDB(context.Background(), bj, resolved)
|
||||
if !errors.Is(err, errBackupNotReadBack) {
|
||||
t.Fatalf("reconcileMariaDB err = %v, want errBackupNotReadBack so the reconcile is retried", err)
|
||||
}
|
||||
persisted := &backupsv1alpha1.BackupJob{}
|
||||
if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Error("the BackupJob was failed although its Backup exists")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileMariaDB_RejectedCreateFailsTheJob pins that only the
|
||||
// read-back is retried: a Create the apiserver rejects still fails the
|
||||
// BackupJob.
|
||||
func TestReconcileMariaDB_RejectedCreateFailsTheJob(t *testing.T) {
|
||||
strategyGroup := strategyv1alpha1.GroupVersion.Group
|
||||
now := metav1.Now()
|
||||
bj := newMariaDBBackupJob("bj-rejected", "tenant")
|
||||
bj.Status.StartedAt = &now
|
||||
bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
|
||||
strategy := &strategyv1alpha1.MariaDB{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"},
|
||||
Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()},
|
||||
}
|
||||
operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}}
|
||||
operatorBackup := &mariadbtypes.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "op-bk",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: bj.Name,
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace,
|
||||
},
|
||||
},
|
||||
Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}},
|
||||
}
|
||||
store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup)
|
||||
r := &BackupJobReconciler{
|
||||
Client: forbidBackupCreate(t, store),
|
||||
Scheme: store.Scheme(),
|
||||
Recorder: record.NewFakeRecorder(10),
|
||||
}
|
||||
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name,
|
||||
}}
|
||||
|
||||
if _, err := r.reconcileMariaDB(context.Background(), bj, resolved); err != nil {
|
||||
t.Fatalf("reconcileMariaDB err = %v, want the BackupJob failed rather than retried", err)
|
||||
}
|
||||
persisted := &backupsv1alpha1.BackupJob{}
|
||||
if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil {
|
||||
t.Fatalf("get BackupJob: %v", err)
|
||||
}
|
||||
if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase)
|
||||
}
|
||||
}
|
||||
@@ -214,7 +214,7 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
|
||||
}
|
||||
artifact, err := r.createMariaDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered)
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
|
||||
return r.failBackupArtifact(ctx, j, err)
|
||||
}
|
||||
now := metav1.Now()
|
||||
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
|
||||
@@ -365,6 +365,10 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
|
||||
mdbBackup *mariadbtypes.Backup,
|
||||
rendered *strategyv1alpha1.MariaDBTemplate,
|
||||
) (*backupsv1alpha1.Backup, error) {
|
||||
existing, probe := r.existingBackup(ctx, j.Namespace, j.Name)
|
||||
if existing != nil {
|
||||
return existing, nil
|
||||
}
|
||||
takenAt := metav1.Now()
|
||||
if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() {
|
||||
takenAt = cond.LastTransitionTime
|
||||
@@ -393,6 +397,9 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
|
||||
// no Artifact at all - leave it nil instead.
|
||||
if uri := driverMD[mariadbStorageURIKey]; uri != "" {
|
||||
status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri}
|
||||
if probe {
|
||||
status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time)
|
||||
}
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -415,8 +422,8 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
|
||||
return nil, err
|
||||
}
|
||||
existing := &backupsv1alpha1.Backup{}
|
||||
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
|
||||
return nil, getErr
|
||||
if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
|
||||
return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr)
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
@@ -558,7 +558,9 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo
|
||||
veleroBackupNamespaceMetadataKey: veleroBackup.Namespace,
|
||||
}
|
||||
|
||||
// Create a basic artifact referencing the Velero backup
|
||||
// SizeBytes stays unset: data lands in a content-addressed kopia
|
||||
// repository shared across backups, so there is no per-backup stored
|
||||
// size, and DataUpload totalBytes counts bytes read, not bytes stored.
|
||||
artifact := &backupsv1alpha1.BackupArtifact{
|
||||
URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{{- /*
|
||||
The ClusterRoleBindings the OIDC bootstrap Job (oidc-rbac-job.yaml) applies
|
||||
inside the tenant cluster, one per `oidc.users` entry, and the list of their
|
||||
names it prunes against. They travel in a ConfigMap the Job mounts and not as
|
||||
text inside its script: the script reaches the container as a single `sh -c`
|
||||
argument, Linux caps one argument at 128 KiB, and a binding per user crosses
|
||||
that cap at about 240 users, after which the container cannot start.
|
||||
|
||||
A release-owned object, not a hook resource, so it exists before the
|
||||
post-install and post-upgrade hooks run. It is rendered only where users are
|
||||
bound; at `mode: None` the Job takes its early branch and the volume is
|
||||
optional.
|
||||
*/}}
|
||||
{{- if ne .Values.oidc.mode "None" }}
|
||||
{{- $desired := "" }}
|
||||
{{- $bindings := "" }}
|
||||
{{- range .Values.oidc.users }}
|
||||
{{- $name := printf "cozystack-oidc-%s" (printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16) }}
|
||||
{{- $role := "view" }}
|
||||
{{- if eq .role "admin" }}
|
||||
{{- $role = "cluster-admin" }}
|
||||
{{- end }}
|
||||
{{- $desired = printf "%s%s\n" $desired $name }}
|
||||
{{- $binding := dict
|
||||
"apiVersion" "rbac.authorization.k8s.io/v1"
|
||||
"kind" "ClusterRoleBinding"
|
||||
"metadata" (dict "name" $name "labels" (dict "app.kubernetes.io/managed-by" "cozystack-oidc" "app.kubernetes.io/instance" $.Release.Name))
|
||||
"roleRef" (dict "apiGroup" "rbac.authorization.k8s.io" "kind" "ClusterRole" "name" $role)
|
||||
"subjects" (list (dict "kind" "User" "apiGroup" "rbac.authorization.k8s.io" "name" .email)) }}
|
||||
{{- $bindings = printf "%s---\n%s\n" $bindings (toYaml $binding) }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-oidc-bindings
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
data:
|
||||
{{- dict "crbs.yaml" $bindings "desired.txt" $desired | toYaml | nindent 2 }}
|
||||
{{- end }}
|
||||
@@ -25,7 +25,9 @@
|
||||
- `mode: System | CustomConfig`:
|
||||
- Apply one ClusterRoleBinding per `users[]` entry inside the tenant
|
||||
cluster (`admin` → `cluster-admin`, `view` → `view`, subject = the
|
||||
literal email matched against the `email` claim).
|
||||
literal email matched against the `email` claim). The bindings are
|
||||
rendered by oidc-rbac-bindings.yaml into a ConfigMap this Job mounts,
|
||||
not into the script below.
|
||||
- Prune stale CRBs labelled by this release.
|
||||
- `mode: System`: patch the pre-rendered `<release>-oidc-kubeconfig`
|
||||
Secret with a real kubeconfig wired to `kubectl oidc-login`
|
||||
@@ -168,6 +170,8 @@ spec:
|
||||
value: {{ $issuer | quote }}
|
||||
- name: OIDC_SERVER_URL
|
||||
value: {{ printf "https://%s:443" $externalHost | quote }}
|
||||
- name: OIDC_BINDINGS_DIR
|
||||
value: /etc/oidc-bindings
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
@@ -254,46 +258,32 @@ spec:
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# A missing list must stop the Job here: grep on a file that is not
|
||||
# there exits 2, `! grep` is then true, and the prune below would
|
||||
# delete every binding of the release.
|
||||
BINDINGS=$OIDC_BINDINGS_DIR
|
||||
if [ ! -f "$BINDINGS/desired.txt" ]; then
|
||||
echo "$BINDINGS/desired.txt is not mounted (ConfigMap $RELEASE-oidc-bindings); refusing to prune against a list that is not there" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Applying ${OIDC_MODE} ClusterRoleBindings to the tenant cluster..."
|
||||
DESIRED_FILE=$(mktemp)
|
||||
{{- range .Values.oidc.users }}
|
||||
{{- $hash := printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16 }}
|
||||
{{- $crbName := printf "cozystack-oidc-%s" $hash }}
|
||||
{{- $role := "view" }}
|
||||
{{- if eq .role "admin" }}
|
||||
{{- $role = "cluster-admin" }}
|
||||
{{- end }}
|
||||
echo "{{ $crbName }}" >> "$DESIRED_FILE"
|
||||
cat <<'EOF' | kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename -
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ $crbName }}
|
||||
labels:
|
||||
app.kubernetes.io/managed-by: cozystack-oidc
|
||||
app.kubernetes.io/instance: {{ $.Release.Name }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ $role }}
|
||||
subjects:
|
||||
- kind: User
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
name: {{ .email | quote }}
|
||||
EOF
|
||||
{{- end }}
|
||||
# kubectl apply fails on a file with no objects, and a release with
|
||||
# no users has none.
|
||||
if [ -s "$BINDINGS/crbs.yaml" ]; then
|
||||
kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename "$BINDINGS/crbs.yaml"
|
||||
fi
|
||||
|
||||
echo "Pruning stale ClusterRoleBindings labelled by this release..."
|
||||
EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \
|
||||
--selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \
|
||||
--output=jsonpath='{.items[*].metadata.name}')
|
||||
for crb in $EXISTING; do
|
||||
if ! grep -x -q -F "$crb" "$DESIRED_FILE"; then
|
||||
if ! grep -x -q -F "$crb" "$BINDINGS/desired.txt"; then
|
||||
echo " removing $crb"
|
||||
kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found
|
||||
fi
|
||||
done
|
||||
rm -f "$DESIRED_FILE"
|
||||
|
||||
if [ "$OIDC_MODE" = "System" ]; then
|
||||
echo "Patching the OIDC kubeconfig Secret on the management cluster..."
|
||||
@@ -349,6 +339,9 @@ spec:
|
||||
- name: kubeconfig
|
||||
mountPath: /etc/kubernetes/kubeconfig
|
||||
readOnly: true
|
||||
- name: oidc-bindings
|
||||
mountPath: /etc/oidc-bindings
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
@@ -356,6 +349,10 @@ spec:
|
||||
secret:
|
||||
secretName: {{ .Release.Name }}-admin-kubeconfig
|
||||
optional: true
|
||||
- name: oidc-bindings
|
||||
configMap:
|
||||
name: {{ .Release.Name }}-oidc-bindings
|
||||
optional: true
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- /*
|
||||
|
||||
@@ -2561,7 +2561,73 @@ tests:
|
||||
value: "true"
|
||||
documentIndex: 3
|
||||
|
||||
- it: bootstrap Job command builds a CRB for each users[] entry
|
||||
- it: the bindings ConfigMap holds a CRB for each users[] entry
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-bindings.yaml
|
||||
set:
|
||||
_cluster.oidc-enabled: "true"
|
||||
_cluster.root-host: example.org
|
||||
oidc.mode: System
|
||||
oidc.users:
|
||||
- email: alice@example.org
|
||||
role: admin
|
||||
- email: bob@example.org
|
||||
role: view
|
||||
asserts:
|
||||
- isKind:
|
||||
of: ConfigMap
|
||||
- equal:
|
||||
path: metadata.name
|
||||
value: kubernetes-app-oidc-bindings
|
||||
- matchRegex:
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'name: alice@example.org'
|
||||
- matchRegex:
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'name: bob@example.org'
|
||||
- matchRegex:
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'app.kubernetes.io/managed-by: cozystack-oidc'
|
||||
- matchRegex:
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'app.kubernetes.io/instance: kubernetes-app'
|
||||
|
||||
- it: the binding names and roles stay what the Job script used to render
|
||||
# The prune step and every binding already in a tenant cluster go by these
|
||||
# names: cozystack-oidc- plus the first 16 hex digits of
|
||||
# sha256("<release>-<email>"). A different name leaves the old binding
|
||||
# behind next to the new one.
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-bindings.yaml
|
||||
set:
|
||||
_cluster.oidc-enabled: "true"
|
||||
_cluster.root-host: example.org
|
||||
oidc.mode: System
|
||||
oidc.users:
|
||||
- email: alice@example.org
|
||||
role: admin
|
||||
- email: bob@example.org
|
||||
role: view
|
||||
asserts:
|
||||
- equal:
|
||||
path: data["desired.txt"]
|
||||
value: "cozystack-oidc-0001a28136ca1a02\ncozystack-oidc-ec8dfb1cbc75e699\n"
|
||||
- matchRegex:
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'name: cozystack-oidc-0001a28136ca1a02\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: alice@example.org\n'
|
||||
- matchRegex:
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'name: cozystack-oidc-ec8dfb1cbc75e699\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: view\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: bob@example.org\n'
|
||||
|
||||
- it: the bootstrap Job mounts the bindings ConfigMap and carries no users[] entry in its script
|
||||
# The script is one `sh -c` argument and Linux caps one argument at 128 KiB:
|
||||
# a binding per user in it stops the container from starting at about 240
|
||||
# users. hack/kubernetes-oidc-bindings.bats renders 300 users to hold the
|
||||
# size itself.
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
@@ -2575,23 +2641,130 @@ tests:
|
||||
role: admin
|
||||
- email: bob@example.org
|
||||
role: view
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.volumes
|
||||
content:
|
||||
name: oidc-bindings
|
||||
configMap:
|
||||
name: kubernetes-app-oidc-bindings
|
||||
optional: true
|
||||
documentIndex: 3
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
content:
|
||||
name: oidc-bindings
|
||||
mountPath: /etc/oidc-bindings
|
||||
readOnly: true
|
||||
documentIndex: 3
|
||||
# The script reads the directory from the environment, which is also how
|
||||
# hack/kubernetes-oidc-bindings.bats points it at a scratch directory.
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: OIDC_BINDINGS_DIR
|
||||
value: /etc/oidc-bindings
|
||||
documentIndex: 3
|
||||
- notMatchRegex:
|
||||
path: spec.template.spec.containers[0].command[2]
|
||||
pattern: 'alice@example\.org'
|
||||
documentIndex: 3
|
||||
- notMatchRegex:
|
||||
path: spec.template.spec.containers[0].command[2]
|
||||
pattern: 'bob@example\.org'
|
||||
documentIndex: 3
|
||||
|
||||
- it: the bootstrap Job stops before pruning when the desired list is not mounted
|
||||
# `! grep -F` on a file that is not there is true, so a missing mount would
|
||||
# delete every binding of the release.
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-job.yaml
|
||||
set:
|
||||
_cluster.oidc-enabled: "true"
|
||||
_cluster.root-host: example.org
|
||||
oidc.mode: System
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].command[2]
|
||||
pattern: 'name: "alice@example.org"'
|
||||
pattern: '(?s)if \[ ! -f "\$BINDINGS/desired\.txt" \]; then.*?exit 1\n\s*fi.*Pruning stale ClusterRoleBindings'
|
||||
documentIndex: 3
|
||||
|
||||
- it: the bootstrap Job applies the bindings file only when it has objects in it
|
||||
# kubectl apply exits non-zero on a file with no objects, and a release
|
||||
# with no users has none.
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-job.yaml
|
||||
set:
|
||||
_cluster.oidc-enabled: "true"
|
||||
_cluster.root-host: example.org
|
||||
oidc.mode: System
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].command[2]
|
||||
pattern: 'name: "bob@example.org"'
|
||||
pattern: 'if \[ -s "\$BINDINGS/crbs\.yaml" \]; then\n\s+kubectl --kubeconfig "\$KUBECONFIG_TENANT" apply --filename "\$BINDINGS/crbs\.yaml"\n\s*fi'
|
||||
documentIndex: 3
|
||||
|
||||
- it: a release with no users renders an empty bindings ConfigMap
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-bindings.yaml
|
||||
set:
|
||||
_cluster.oidc-enabled: "true"
|
||||
_cluster.root-host: example.org
|
||||
oidc.mode: System
|
||||
oidc.users: []
|
||||
asserts:
|
||||
- equal:
|
||||
path: data["crbs.yaml"]
|
||||
value: ""
|
||||
- equal:
|
||||
path: data["desired.txt"]
|
||||
value: ""
|
||||
|
||||
- it: CustomConfig mode renders the bindings ConfigMap for its users
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-bindings.yaml
|
||||
set:
|
||||
_cluster.oidc-enabled: "false"
|
||||
oidc.mode: CustomConfig
|
||||
oidc.customConfig.config: |
|
||||
apiVersion: apiserver.config.k8s.io/v1beta1
|
||||
kind: AuthenticationConfiguration
|
||||
jwt:
|
||||
- issuer:
|
||||
url: https://idp.example.com
|
||||
audiences: [my-cluster]
|
||||
oidc.users:
|
||||
- email: alice@example.org
|
||||
role: admin
|
||||
asserts:
|
||||
- isKind:
|
||||
of: ConfigMap
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].command[2]
|
||||
pattern: 'name: cluster-admin'
|
||||
documentIndex: 3
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].command[2]
|
||||
pattern: 'name: view'
|
||||
documentIndex: 3
|
||||
path: data["crbs.yaml"]
|
||||
pattern: 'name: alice@example.org'
|
||||
|
||||
- it: mode=None renders no bindings ConfigMap even when users are set
|
||||
# The Job takes its early branch at mode None and never reads the file.
|
||||
release:
|
||||
name: kubernetes-app
|
||||
namespace: tenant-foo
|
||||
template: templates/oidc-rbac-bindings.yaml
|
||||
set:
|
||||
oidc.mode: None
|
||||
oidc.users:
|
||||
- email: alice@example.org
|
||||
role: admin
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: mode=System aborts the Job instead of leaving the OIDC kubeconfig Secret as the install-time placeholder when the tenant CA is missing
|
||||
# A placeholder kubeconfig cannot authenticate, so the release must not go Ready with it.
|
||||
|
||||
+9
-2
@@ -131,8 +131,15 @@ spec:
|
||||
For example: "sha256:<hex>".
|
||||
type: string
|
||||
sizeBytes:
|
||||
description: SizeBytes is the size of the artifact in bytes, if
|
||||
known.
|
||||
description: |-
|
||||
SizeBytes is the stored size in bytes of the files this backup owns,
|
||||
if known: the dump of a MariaDB backup, for example, or the base backup
|
||||
of a CNPG one, which leaves out the WAL archive the server's backups
|
||||
share and a point-in-time restore also reads. Strategies that cannot
|
||||
attribute stored bytes to a single backup, such as Velero's, whose
|
||||
repository deduplicates across backups, leave it unset. Drivers never
|
||||
write 0 for a size they cannot measure, so an absent value means
|
||||
"unknown", not "empty".
|
||||
format: int64
|
||||
type: integer
|
||||
uri:
|
||||
|
||||
+46
-17
@@ -3,8 +3,8 @@
|
||||
1. Helm renders this file at chart install time. Anything inside
|
||||
`{{ ... }}` is interpreted by Helm.
|
||||
2. The strategy controller renders the resulting Strategy CR
|
||||
per-BackupJob, expanding `{{ .Application | .Release | .Mode }}`
|
||||
against the live application object.
|
||||
per-BackupJob, expanding `{{ .Application | .Release | .Mode |
|
||||
.ArchiveName | .Backup }}` against the live application object.
|
||||
To emit a literal "{{ .Foo }}" that survives Helm, we pass it as the
|
||||
string argument of `printf` — Helm does not interpret inner template
|
||||
delimiters inside printf's quoted strings. Do NOT simplify by inlining
|
||||
@@ -51,6 +51,11 @@ spec:
|
||||
host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc"
|
||||
api="http://${host}:7171"
|
||||
auth="-u ${API_USERNAME}:${API_PASSWORD}"
|
||||
# The archive the controller named for this BackupJob and records
|
||||
# on its Backup. The default renders it empty, rather than as
|
||||
# text/template's missing-key marker, when a controller that
|
||||
# predates archive names renders this template during an upgrade.
|
||||
archive="{{ printf `{{ .ArchiveName | default "" }}` }}"
|
||||
# Every request but the POST is retried on failure, every 5s,
|
||||
# for $2 attempts in all: once the operation is accepted, a
|
||||
# dropped poll must not fail the attempt, since the Job retry it
|
||||
@@ -91,23 +96,47 @@ spec:
|
||||
}
|
||||
if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then
|
||||
prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-"
|
||||
# The name is new on every attempt, so a retry never takes over
|
||||
# the create_remote of the attempt it replaces; that needs a
|
||||
# name fixed for the whole BackupJob.
|
||||
target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
cmd="create_remote ${target}"
|
||||
# Only a controller that predates archive names leaves it
|
||||
# empty, and that one records no name on the Backup either.
|
||||
target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
# A retried Pod gets the same name, so it takes over an attempt
|
||||
# the sidecar is still running, and otherwise asks for resume,
|
||||
# so a local or partial remote backup an earlier attempt left is
|
||||
# resumed rather than refused as existing, whatever
|
||||
# use_resumable_state the sidecar runs with. The sidecar logs
|
||||
# the request as "create_remote --resume <name>".
|
||||
cmd="create_remote --resume ${target}"
|
||||
wait_turn "${cmd}"
|
||||
echo "${cmd}"
|
||||
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null
|
||||
if [ "${turn}" = "join" ]; then
|
||||
echo "${cmd} is already running; waiting for it"
|
||||
else
|
||||
echo "${cmd}"
|
||||
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null
|
||||
fi
|
||||
else
|
||||
prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-"
|
||||
listing=$(api_get /backup/list/remote 60)
|
||||
target=$(printf '%s\n' "${listing}" \
|
||||
| jq -rs --arg prefix "${prefix}" \
|
||||
'[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""')
|
||||
if [ -z "${target}" ] || [ "${target}" = "null" ]; then
|
||||
echo "no remote backup found" >&2
|
||||
exit 1
|
||||
app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}"
|
||||
backup="{{ printf "{{ .Backup.Name }}" }}"
|
||||
prefix="clickhouse-${app}-"
|
||||
remote=$(api_get /backup/list/remote 60)
|
||||
if [ -n "${archive}" ]; then
|
||||
if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then
|
||||
echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
target="${archive}"
|
||||
else
|
||||
# takenAt is set after the backup Job finished, so the
|
||||
# archive that Job wrote is stamped no later than it.
|
||||
notafter="{{ printf `{{ .Backup.TakenAt | default "" }}` }}"
|
||||
echo "WARNING: Backup ${backup} records no archive name; restoring the newest archive of ${app} stamped no later than its takenAt (${notafter}), a best-effort choice that may not be the archive this Backup was taken as" >&2
|
||||
target=$(printf '%s\n' "${remote}" | jq -rs --arg prefix "${prefix}" --arg notafter "${notafter}" '
|
||||
[.[] | select((.desc // "") | startswith("broken") | not) | .name // ""
|
||||
| select(startswith($prefix) and (ltrimstr($prefix) | test("^[0-9]{8}T[0-9]{6}Z$")) and ltrimstr($prefix) <= $notafter)]
|
||||
| sort | last // ""')
|
||||
if [ -z "${target}" ]; then
|
||||
echo "no archive of ${app} stamped no later than ${notafter} in remote storage" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
# restore_remote drops every table it restores before
|
||||
# recreating it, so a second run discards whatever was written
|
||||
|
||||
@@ -240,10 +240,12 @@ exclude_from_kube_ovn() {
|
||||
fi
|
||||
# A pod that holds a node's current Ingress IP keeps the address after the
|
||||
# move, and that node keeps it in its ipcache as reserved:ingress, so the pod
|
||||
# stays unreachable from there until it is recreated.
|
||||
jq -r --argjson cn "$ciliumnodes" '
|
||||
([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress
|
||||
| .[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ipobjects" \
|
||||
# stays unreachable from there until it is recreated. The CiliumNode list goes
|
||||
# in on stdin, because the Node labels Cilium copies onto every CiliumNode take
|
||||
# it past the 128 KiB a single argument may hold within a handful of nodes.
|
||||
jq -rn 'input as $cn
|
||||
| ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress
|
||||
| input[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ciliumnodes"$'\n'"$ipobjects" \
|
||||
| while read -r ipname ip node; do
|
||||
if ! in_any "$POOL" "$ip"; then
|
||||
log "$ipname holds $ip, the Ingress IP of $node before the move. Recreate it once the run is done, $node keeps that address as reserved:ingress"
|
||||
|
||||
@@ -55,6 +55,23 @@ type ApplicationStatus struct {
|
||||
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
|
||||
// +optional
|
||||
ExternalIPsCount int32 `json:"externalIPsCount,omitempty"`
|
||||
// InstallFailures is the install failure count of the underlying HelmRelease.
|
||||
// It counts only attempts that recorded a Helm release, so a failure before
|
||||
// Helm records one, such as a chart that fails to render, shows only as
|
||||
// Ready=False; a failure to apply the rendered resources is counted. It
|
||||
// resets when the desired state (generation, chart version, values) changes,
|
||||
// and on success only under the RetryOnFailure strategy, so a Ready release
|
||||
// may carry it.
|
||||
// +optional
|
||||
InstallFailures int64 `json:"installFailures,omitempty"`
|
||||
// UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
|
||||
// with the same counting and reset rules as InstallFailures.
|
||||
// +optional
|
||||
UpgradeFailures int64 `json:"upgradeFailures,omitempty"`
|
||||
// LastAttemptedReleaseAction is the last release action (install or upgrade)
|
||||
// the underlying HelmRelease attempted.
|
||||
// +optional
|
||||
LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"`
|
||||
}
|
||||
|
||||
// SchedulingClass returns the scheduling class requested by this Application.
|
||||
|
||||
@@ -34,6 +34,20 @@ type ApplicationStatusApplyConfiguration struct {
|
||||
Namespace *string `json:"namespace,omitempty"`
|
||||
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
|
||||
ExternalIPsCount *int32 `json:"externalIPsCount,omitempty"`
|
||||
// InstallFailures is the install failure count of the underlying HelmRelease.
|
||||
// It counts only attempts that recorded a Helm release, so a failure before
|
||||
// Helm records one, such as a chart that fails to render, shows only as
|
||||
// Ready=False; a failure to apply the rendered resources is counted. It
|
||||
// resets when the desired state (generation, chart version, values) changes,
|
||||
// and on success only under the RetryOnFailure strategy, so a Ready release
|
||||
// may carry it.
|
||||
InstallFailures *int64 `json:"installFailures,omitempty"`
|
||||
// UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
|
||||
// with the same counting and reset rules as InstallFailures.
|
||||
UpgradeFailures *int64 `json:"upgradeFailures,omitempty"`
|
||||
// LastAttemptedReleaseAction is the last release action (install or upgrade)
|
||||
// the underlying HelmRelease attempted.
|
||||
LastAttemptedReleaseAction *string `json:"lastAttemptedReleaseAction,omitempty"`
|
||||
}
|
||||
|
||||
// ApplicationStatusApplyConfiguration constructs a declarative configuration of the ApplicationStatus type for use with
|
||||
@@ -78,3 +92,27 @@ func (b *ApplicationStatusApplyConfiguration) WithExternalIPsCount(value int32)
|
||||
b.ExternalIPsCount = &value
|
||||
return b
|
||||
}
|
||||
|
||||
// WithInstallFailures sets the InstallFailures field in the declarative configuration to the given value
|
||||
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
|
||||
// If called multiple times, the InstallFailures field is set to the value of the last call.
|
||||
func (b *ApplicationStatusApplyConfiguration) WithInstallFailures(value int64) *ApplicationStatusApplyConfiguration {
|
||||
b.InstallFailures = &value
|
||||
return b
|
||||
}
|
||||
|
||||
// WithUpgradeFailures sets the UpgradeFailures field in the declarative configuration to the given value
|
||||
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
|
||||
// If called multiple times, the UpgradeFailures field is set to the value of the last call.
|
||||
func (b *ApplicationStatusApplyConfiguration) WithUpgradeFailures(value int64) *ApplicationStatusApplyConfiguration {
|
||||
b.UpgradeFailures = &value
|
||||
return b
|
||||
}
|
||||
|
||||
// WithLastAttemptedReleaseAction sets the LastAttemptedReleaseAction field in the declarative configuration to the given value
|
||||
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
|
||||
// If called multiple times, the LastAttemptedReleaseAction field is set to the value of the last call.
|
||||
func (b *ApplicationStatusApplyConfiguration) WithLastAttemptedReleaseAction(value string) *ApplicationStatusApplyConfiguration {
|
||||
b.LastAttemptedReleaseAction = &value
|
||||
return b
|
||||
}
|
||||
@@ -290,6 +290,27 @@ func schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref common.ReferenceCallbac
|
||||
Format: "int32",
|
||||
},
|
||||
},
|
||||
"installFailures": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "InstallFailures is the install failure count of the underlying HelmRelease. It counts only attempts that recorded a Helm release, so a failure before Helm records one, such as a chart that fails to render, shows only as Ready=False; a failure to apply the rendered resources is counted. It resets when the desired state (generation, chart version, values) changes, and on success only under the RetryOnFailure strategy, so a Ready release may carry it.",
|
||||
Type: []string{"integer"},
|
||||
Format: "int64",
|
||||
},
|
||||
},
|
||||
"upgradeFailures": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "UpgradeFailures is the upgrade failure count of the underlying HelmRelease, with the same counting and reset rules as InstallFailures.",
|
||||
Type: []string{"integer"},
|
||||
Format: "int64",
|
||||
},
|
||||
},
|
||||
"lastAttemptedReleaseAction": {
|
||||
SchemaProps: spec.SchemaProps{
|
||||
Description: "LastAttemptedReleaseAction is the last release action (install or upgrade) the underlying HelmRelease attempted.",
|
||||
Type: []string{"string"},
|
||||
Format: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -1573,13 +1573,23 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H
|
||||
},
|
||||
Spec: filteredSpec,
|
||||
Status: appsv1alpha1.ApplicationStatus{
|
||||
Version: hr.Status.LastAttemptedRevision,
|
||||
Version: hr.Status.LastAttemptedRevision,
|
||||
InstallFailures: hr.Status.InstallFailures,
|
||||
UpgradeFailures: hr.Status.UpgradeFailures,
|
||||
LastAttemptedReleaseAction: string(hr.Status.LastAttemptedReleaseAction),
|
||||
},
|
||||
}
|
||||
|
||||
var conditions []metav1.Condition
|
||||
for _, hrCondition := range hr.GetConditions() {
|
||||
if hrCondition.Type == "Ready" || hrCondition.Type == "Released" {
|
||||
// The Application carries no generation, so a client cannot tell a Stalled
|
||||
// left over from the previous spec from one about the current spec. Compare
|
||||
// the condition's own ObservedGeneration, not Status.ObservedGeneration: the
|
||||
// question is which spec this condition describes.
|
||||
if hrCondition.Type == "Stalled" && hrCondition.ObservedGeneration < hr.Generation {
|
||||
continue
|
||||
}
|
||||
if hrCondition.Type == "Ready" || hrCondition.Type == "Released" || hrCondition.Type == "Stalled" {
|
||||
conditions = append(conditions, metav1.Condition{
|
||||
LastTransitionTime: hrCondition.LastTransitionTime,
|
||||
Reason: hrCondition.Reason,
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
helmv2 "github.com/fluxcd/helm-controller/api/v2"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestConvertReleaseFailure_StalledAndCountersExposed(t *testing.T) {
|
||||
r := newTestRESTWithSchemes()
|
||||
hr := makeHelmRelease("postgresql-mydb", "default")
|
||||
hr.Generation = 2
|
||||
hr.Status.Conditions = []metav1.Condition{
|
||||
{Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 2},
|
||||
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
|
||||
{Type: "Released", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
|
||||
}
|
||||
hr.Status.Failures = 7
|
||||
hr.Status.InstallFailures = 1
|
||||
hr.Status.UpgradeFailures = 3
|
||||
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
|
||||
|
||||
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
sc := findCondition(app.GetConditions(), "Stalled")
|
||||
if sc == nil {
|
||||
t.Fatal("expected Stalled condition to be copied from the HelmRelease")
|
||||
}
|
||||
if sc.Status != metav1.ConditionTrue || sc.Reason != "RetriesExceeded" {
|
||||
t.Errorf("expected Stalled=True/RetriesExceeded, got %s/%s", sc.Status, sc.Reason)
|
||||
}
|
||||
if app.Status.InstallFailures != 1 {
|
||||
t.Errorf("expected installFailures=1, got %d", app.Status.InstallFailures)
|
||||
}
|
||||
if app.Status.UpgradeFailures != 3 {
|
||||
t.Errorf("expected upgradeFailures=3, got %d", app.Status.UpgradeFailures)
|
||||
}
|
||||
if app.Status.LastAttemptedReleaseAction != "install" {
|
||||
t.Errorf("expected lastAttemptedReleaseAction=install, got %q", app.Status.LastAttemptedReleaseAction)
|
||||
}
|
||||
}
|
||||
|
||||
// After a spec update the HelmRelease keeps the Stalled condition of the
|
||||
// previous generation until helm-controller reconciles it; reported as is, it
|
||||
// would read as a terminal failure of the new values.
|
||||
func TestConvertReleaseFailure_StalledFromOlderGenerationOmitted(t *testing.T) {
|
||||
r := newTestRESTWithSchemes()
|
||||
hr := makeHelmRelease("postgresql-mydb", "default")
|
||||
hr.Generation = 2
|
||||
hr.Status.Conditions = []metav1.Condition{
|
||||
{Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 1},
|
||||
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 1},
|
||||
}
|
||||
hr.Status.InstallFailures = 1
|
||||
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
|
||||
|
||||
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if findCondition(app.GetConditions(), "Stalled") != nil {
|
||||
t.Error("expected Stalled observed at an older generation not to be copied")
|
||||
}
|
||||
if findCondition(app.GetConditions(), "Ready") == nil {
|
||||
t.Error("expected Ready to still be copied")
|
||||
}
|
||||
}
|
||||
|
||||
// helm-controller clears the counters on success only under the RetryOnFailure
|
||||
// strategy, so a Ready release can still carry them; they must not be hidden.
|
||||
func TestConvertReleaseFailure_CountersKeptOnReadyRelease(t *testing.T) {
|
||||
r := newTestRESTWithSchemes()
|
||||
hr := makeHelmRelease("postgresql-mydb", "default")
|
||||
hr.Status.Conditions = []metav1.Condition{
|
||||
{Type: "Ready", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
|
||||
{Type: "Released", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
|
||||
}
|
||||
hr.Status.InstallFailures = 2
|
||||
hr.Status.UpgradeFailures = 1
|
||||
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionUpgrade
|
||||
|
||||
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if findCondition(app.GetConditions(), "Stalled") != nil {
|
||||
t.Error("expected no Stalled condition on a release that does not carry one")
|
||||
}
|
||||
if app.Status.InstallFailures != 2 || app.Status.UpgradeFailures != 1 {
|
||||
t.Errorf("expected install=2 upgrade=1, got install=%d upgrade=%d", app.Status.InstallFailures, app.Status.UpgradeFailures)
|
||||
}
|
||||
if app.Status.LastAttemptedReleaseAction != "upgrade" {
|
||||
t.Errorf("expected lastAttemptedReleaseAction=upgrade, got %q", app.Status.LastAttemptedReleaseAction)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertReleaseFailure_ReconcilingNotCopied(t *testing.T) {
|
||||
r := newTestRESTWithSchemes()
|
||||
hr := makeHelmRelease("postgresql-mydb", "default")
|
||||
hr.Status.Conditions = []metav1.Condition{
|
||||
{Type: "Reconciling", Status: metav1.ConditionTrue, Reason: "Progressing", Message: "Running 'install' action"},
|
||||
{Type: "Ready", Status: metav1.ConditionUnknown, Reason: "Progressing", Message: "Running 'install' action"},
|
||||
}
|
||||
|
||||
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if findCondition(app.GetConditions(), "Reconciling") != nil {
|
||||
t.Error("expected Reconciling not to be copied")
|
||||
}
|
||||
if findCondition(app.GetConditions(), "Ready") == nil {
|
||||
t.Error("expected Ready to be copied")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user