diff --git a/api/backups/v1alpha1/backup_types.go b/api/backups/v1alpha1/backup_types.go index fd76ac47e..483bb1fd8 100644 --- a/api/backups/v1alpha1/backup_types.go +++ b/api/backups/v1alpha1/backup_types.go @@ -37,7 +37,9 @@ type BackupArtifact struct { // For example: s3://bucket/prefix/file.tar.gz URI string `json:"uri"` - // SizeBytes is the size of the artifact in bytes, if known. + // SizeBytes is the size of the artifact in bytes, if known. Drivers + // leave it unset rather than writing 0 when they cannot measure the + // stored size, so an absent value means "unknown", not "empty". // +optional SizeBytes int64 `json:"sizeBytes,omitempty"` diff --git a/go.mod b/go.mod index 6109faf15..34352e583 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/minio/minio-go/v7 v7.0.98 github.com/onsi/ginkgo/v2 v2.33.0 github.com/onsi/gomega v1.43.1 github.com/prometheus/client_golang v1.24.1 @@ -63,6 +64,7 @@ require ( github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.4 // indirect github.com/gkampitakis/go-snaps v0.5.23 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v1.0.1 // indirect github.com/go-openapi/jsonreference v1.0.2 // indirect @@ -93,18 +95,25 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.71.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sirupsen/logrus v1.10.2 // indirect github.com/tidwall/match v1.2.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/api/v3 v3.7.0 // indirect go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect diff --git a/go.sum b/go.sum index dea6aaf08..667ca71ee 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg= github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -141,6 +143,11 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -151,6 +158,12 @@ github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -165,6 +178,8 @@ github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8 github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.43.1 h1:vGIPFuYrIO6/0Z09s0I0QQQgFchiX4+tb1re3MScJYo= github.com/onsi/gomega v1.43.1/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -180,6 +195,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= @@ -206,6 +223,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vmware-tanzu/velero v1.18.3 h1:9uwN5yxTWQl9PsF5ljX4xiG+7VIq3I9mxOQ0DyyZ31A= diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index c6b743356..f329eed4d 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -39,6 +39,10 @@ type BackupJobReconciler struct { // re-reads the BackupJob before dispatching to a driver. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to measure artifact + // sizes; nil means a real minio client. Set by unit tests. + newS3ObjectLister func(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) } func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go new file mode 100644 index 000000000..b968a98d4 --- /dev/null +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSizeProbeTimeout = 30 * time.Second + + // Slack around the [created, completed] window of the operator Backup + // when matching the timestamp mariadb-operator embeds in the dump file + // name: that timestamp comes from the backup Job pod's clock, not the + // apiserver's, and is truncated to seconds. + mariadbSizeProbeClockSkew = time.Minute +) + +type s3ObjectInfo struct { + Key string + Size int64 +} + +type s3ObjectLister interface { + ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) +} + +// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for +// mdbBackup. The strategy prefix is shared by every backup of the +// application, so the dump is identified by the timestamp in its file name +// falling inside the operator Backup's lifetime. Returns 0 - leaving +// Artifact.SizeBytes absent - when the object store cannot be read or the +// window does not match exactly one dump: an unknown size must not be +// reported as a measured one. +func (r *BackupJobReconciler) mariadbArtifactSizeBytes( + ctx context.Context, + namespace string, + s3 *strategyv1alpha1.MariaDBS3Template, + mdbBackup *mariadbtypes.Backup, + completedAt time.Time, +) int64 { + logger := getLogger(ctx) + ctx, cancel := context.WithTimeout(ctx, mariadbSizeProbeTimeout) + defer cancel() + + newLister := r.newS3ObjectLister + if newLister == nil { + newLister = r.newMariaDBMinioLister + } + lister, err := newLister(ctx, namespace, s3) + if err != nil { + logger.Debug("cannot build S3 client to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) + return 0 + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + objects, err := lister.ListObjects(ctx, s3.Bucket, prefix) + if err != nil { + logger.Debug("cannot list MariaDB backup objects to measure size", "backup", mdbBackup.Name, "error", err) + return 0 + } + + lower := mdbBackup.CreationTimestamp.Add(-mariadbSizeProbeClockSkew) + upper := completedAt.Add(mariadbSizeProbeClockSkew) + var matched []s3ObjectInfo + for _, obj := range objects { + ts, ok := parseMariaDBLogicalBackupFile(strings.TrimPrefix(obj.Key, prefix)) + if !ok || ts.Before(lower) || ts.After(upper) { + continue + } + matched = append(matched, obj) + } + if len(matched) != 1 { + logger.Debug("MariaDB backup dump not attributable to a single object; leaving size unset", + "backup", mdbBackup.Name, "candidates", len(matched)) + return 0 + } + return matched[0].Size +} + +// mariadbOperatorS3Prefix mirrors how mariadb-operator normalises +// spec.storage.s3.prefix before prepending it to object keys. +func mariadbOperatorS3Prefix(prefix string) string { + if prefix == "" || prefix == "/" { + return "" + } + if !strings.HasSuffix(prefix, "/") { + return prefix + "/" + } + return prefix +} + +// parseMariaDBLogicalBackupFile accepts the logical-backup file names +// mariadb-operator writes, backup..sql and +// backup...sql, and returns the embedded timestamp. +func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) { + parts := strings.Split(name, ".") + if len(parts) != 3 && len(parts) != 4 { + return time.Time{}, false + } + if parts[0] != "backup" || parts[len(parts)-1] != "sql" { + return time.Time{}, false + } + ts, err := time.Parse(time.RFC3339, parts[1]) + if err != nil { + return time.Time{}, false + } + return ts, true +} + +type minioObjectLister struct { + client *minio.Client +} + +func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { + var out []s3ObjectInfo + for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) { + if obj.Err != nil { + return nil, obj.Err + } + out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size}) + } + return out, nil +} + +// newMariaDBMinioLister builds an S3 client from the same Secrets and TLS +// settings the operator Backup was rendered with, so the controller reads +// the bucket with the tenant's credentials rather than its own. +func (r *BackupJobReconciler) newMariaDBMinioLister(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) { + accessKey, err := r.mariadbSecretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef) + if err != nil { + return nil, err + } + secretKey, err := r.mariadbSecretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef) + if err != nil { + return nil, err + } + var sessionToken string + if s3.SessionTokenSecretKeyRef != nil { + if sessionToken, err = r.mariadbSecretValue(ctx, namespace, *s3.SessionTokenSecretKeyRef); err != nil { + return nil, err + } + } + + opts := &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, sessionToken), + Region: s3.Region, + Secure: s3.TLS != nil && s3.TLS.Enabled, + } + if opts.Secure && s3.TLS.CASecretKeyRef != nil { + caPEM, err := r.mariadbSecretValue(ctx, namespace, *s3.TLS.CASecretKeyRef) + if err != nil { + return nil, err + } + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM([]byte(caPEM)) { + return nil, fmt.Errorf("no PEM certificates in Secret %s/%s key %q", namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) + } + opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}} + } + client, err := minio.New(s3.Endpoint, opts) + if err != nil { + return nil, err + } + return &minioObjectLister{client: client}, nil +} + +func (r *BackupJobReconciler) mariadbSecretValue(ctx context.Context, namespace string, ref strategyv1alpha1.MariaDBSecretKeySelector) (string, error) { + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: ref.Name}, secret); err != nil { + return "", err + } + v, ok := secret.Data[ref.Key] + if !ok { + return "", fmt.Errorf("Secret %s/%s has no key %q", namespace, ref.Name, ref.Key) + } + return string(v), nil +} diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go new file mode 100644 index 000000000..f7e450a26 --- /dev/null +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +type fakeS3Lister struct { + objects []s3ObjectInfo + err error + + bucket, prefix string +} + +func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { + f.bucket, f.prefix = bucket, prefix + return f.objects, f.err +} + +func reconcilerWithLister(l *fakeS3Lister) *BackupJobReconciler { + return &BackupJobReconciler{ + newS3ObjectLister: func(context.Context, string, *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) { + return l, nil + }, + } +} + +func dumpKey(prefix string, ts time.Time, compression string) string { + if compression == "" { + return fmt.Sprintf("%sbackup.%s.sql", prefix, ts.UTC().Format(time.RFC3339)) + } + return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) +} + +func TestParseMariaDBLogicalBackupFile(t *testing.T) { + cases := map[string]bool{ + "backup.2026-10-21T10:00:00Z.sql": true, + "backup.2026-10-21T10:00:00Z.gzip.sql": true, + "backup.2026-10-21T10:00:00+04:00.sql": true, + "backup.2026-10-21T10:00:00Z.sql.tmp": false, + "physicalbackup-20261021100000.xb": false, + "backup.not-a-date.sql": false, + "sub/backup.2026-10-21T10:00:00Z.sql": false, + "backup.2026-10-21T10:00:00Z.a.gzip.sql": false, + "backups.2026-10-21T10:00:00Z.gzip.sql": false, + "backup.2026-10-21T10:00:00Z.gzip.dump": false, + "": false, + } + for name, want := range cases { + if _, got := parseMariaDBLogicalBackupFile(name); got != want { + t.Errorf("parseMariaDBLogicalBackupFile(%q) = %v, want %v", name, got, want) + } + } +} + +func TestMariaDBOperatorS3Prefix(t *testing.T) { + cases := map[string]string{"": "", "/": "", "ns/app": "ns/app/", "ns/app/": "ns/app/"} + for in, want := range cases { + if got := mariadbOperatorS3Prefix(in); got != want { + t.Errorf("mariadbOperatorS3Prefix(%q) = %q, want %q", in, got, want) + } + } +} + +func TestMariaDBArtifactSizeBytes(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + mdbBackup := &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created), + }} + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + const prefix = "tenant/db/" + + t.Run("single dump inside the backup window", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111}, + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242}, + {Key: prefix + "0-backup-target.txt", Size: 7}, + }} + got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed) + if got != 4242 { + t.Errorf("size = %d, want 4242", got) + } + if l.bucket != "bkt" || l.prefix != prefix { + t.Errorf("listed %q/%q, want bkt/%q", l.bucket, l.prefix, prefix) + } + }) + + t.Run("pod clock skew within tolerance", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9}, + }} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 9 { + t.Errorf("size = %d, want 9", got) + } + }) + + t.Run("two dumps in the window leave size unset", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1}, + {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2}, + }} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0 (ambiguous)", got) + } + }) + + t.Run("no dump in the window leaves size unset", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1}, + }} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + + t.Run("list error leaves size unset", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("AccessDenied")} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + + t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { + c := newMariaDBStrategyTestClient(t) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) +} + +// TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client +// against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and +// response parsing are exercised together. +func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + key := dumpKey("tenant/db/", created.Add(time.Minute), "gzip") + + var gotPath, gotPrefix, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, ` + + bkttenant/db/11000false + %s31337"x"2026-10-21T10:01:00.000Z +`, key) + })) + defer srv.Close() + + creds := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"}, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("AKID"), + "AWS_SECRET_ACCESS_KEY": []byte("secret"), + }, + } + c := newMariaDBStrategyTestClient(t, creds) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Endpoint = strings.TrimPrefix(srv.URL, "http://") + s3.Prefix = "tenant/db/" + s3.Region = "us-east-1" + mdbBackup := &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created), + }} + + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, created.Add(3*time.Minute)); got != 31337 { + t.Fatalf("size = %d, want 31337", got) + } + if gotPath != "/bkt/" || gotPrefix != "tenant/db/" { + t.Errorf("request path=%q prefix=%q, want /bkt/ and tenant/db/", gotPath, gotPrefix) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } +} + +func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { + apps := mariadbapp.GroupName + completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC)) + mdbBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "src-op-bk", + CreationTimestamp: metav1.NewTime(completed.Add(-3 * time.Minute)), + }, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: completed, + }}}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + c := newMariaDBStrategyTestClient(t, job) + r := reconcilerWithLister(&fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048}, + }}) + r.Client, r.Scheme = c, c.Scheme() + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + + artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 { + t.Fatalf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index a5bd77094..84fe0c2df 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -392,7 +392,10 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( // operator's PVC ID), so an Artifact{URI: ""} is a worse signal than // no Artifact at all - leave it nil instead. if uri := driverMD[mariadbStorageURIKey]; uri != "" { - status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} + status.Artifact = &backupsv1alpha1.BackupArtifact{ + URI: uri, + SizeBytes: r.mariadbArtifactSizeBytes(ctx, j.Namespace, rendered.Storage.S3, mdbBackup, takenAt.Time), + } } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/backupcontroller/velerostrategy_controller.go b/internal/backupcontroller/velerostrategy_controller.go index e08b2f818..abedc97be 100644 --- a/internal/backupcontroller/velerostrategy_controller.go +++ b/internal/backupcontroller/velerostrategy_controller.go @@ -558,7 +558,9 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo veleroBackupNamespaceMetadataKey: veleroBackup.Namespace, } - // Create a basic artifact referencing the Velero backup + // SizeBytes stays unset: data lands in a content-addressed kopia + // repository shared across backups, so there is no per-backup stored + // size, and DataUpload totalBytes counts bytes read, not bytes stored. artifact := &backupsv1alpha1.BackupArtifact{ URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name), } diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index 28ef57fb7..f30d3dbc7 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -131,8 +131,10 @@ spec: For example: "sha256:". type: string sizeBytes: - description: SizeBytes is the size of the artifact in bytes, if - known. + description: |- + SizeBytes is the size of the artifact in bytes, if known. Drivers + leave it unset rather than writing 0 when they cannot measure the + stored size, so an absent value means "unknown", not "empty". format: int64 type: integer uri: