diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 9197f07be..40d827c62 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -38,6 +38,11 @@ type s3ObjectLister interface { ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) } +type s3ObjectRemover interface { + // RemoveObject deletes key; a key that does not exist is not an error. + RemoveObject(ctx context.Context, bucket, key string) error +} + // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { @@ -63,8 +68,10 @@ type objectStore struct { // reader reads Secrets, which the manager's client fetches uncached, // and lists operator Backups, which it serves from the cache. reader client.Reader - // newLister overrides the S3 client; nil means a real minio client. - newLister func(conn s3Connection) (s3ObjectLister, error) + // newLister and newRemover override the S3 client; nil means a real + // minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) + newRemover func(conn s3Connection) (s3ObjectRemover, error) } func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { @@ -74,6 +81,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { return newMinioObjectLister(conn) } +func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) { + if s.newRemover != nil { + return s.newRemover(conn) + } + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { @@ -94,6 +112,10 @@ func (r *RestoreJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *BackupReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } @@ -103,6 +125,14 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + +func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) { opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -152,6 +182,10 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } +func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error { + return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}) +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..f817e696b 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -39,9 +39,16 @@ type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds - // into the Backup's namespace before spawning its best-effort delete Job. + // CredentialsConfig locates the platform backup credentials: the Job-based + // cleanups project them into the Backup's namespace, and the MariaDB + // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + + // newS3ObjectLister and newS3ObjectRemover override the S3 client the + // MariaDB cleanup deletes a dump with; nil means a real minio client. + // Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) + newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error) } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -126,22 +133,18 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: - // Cozystack Backup deletion does NOT delete the operator-side - // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. - // Lifecycle of that CR is owned by mariadb-operator's - // MaxRetention setting and by tenants / explicit teardown flows; - // the controller's RBAC has no delete verb on - // k8s.mariadb.com/backups for that reason. Surfacing this as an - // explicit branch (rather than falling through to the Velero - // default) keeps the contract honest: the MariaDB driver does - // not own the archive, so it does not delete it on CR removal. - // Mirrors the Altinity branch's "we do not own S3" stance. - return ctrl.Result{}, nil + // mariadb-operator's maxRetention prunes only from inside the next + // backup run of the same prefix and a Plan never deletes a Backup, so + // a dump on the platform bucket is deleted here or never. A dump in + // a tenant's own bucket is deleted too: the Backup proves it owns + // it, and the tenant deleting the Backup is asking for exactly that + // (see cleanupMariaDBBackup). + return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the - // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup + // archive" contract as Altinity / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer @@ -154,7 +157,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own - // the archive" contract as Altinity / MariaDB — the explicit + // the archive" contract as Altinity — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. diff --git a/internal/backupcontroller/backup_controller_test.go b/internal/backupcontroller/backup_controller_test.go index 579cb5fe6..d77f5d208 100644 --- a/internal/backupcontroller/backup_controller_test.go +++ b/internal/backupcontroller/backup_controller_test.go @@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) { // TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same // invariant as the Altinity test above for the MariaDB strategy. The -// dispatcher must explicitly route MariaDB-strategy Backups to a no-op -// cleanup rather than fall through to the Velero default - the MariaDB -// driver does not own the operator-side k8s.mariadb.com/Backup CR or the -// S3/PVC archive behind it (rbac.yaml grants no delete verb on -// k8s.mariadb.com/backups for that reason). The test seeds the Velero +// dispatcher must route MariaDB-strategy Backups to their own cleanup +// rather than fall through to the Velero default. The test seeds the Velero // metadata keys plus a matching velero.io/Backup; with the explicit branch // no DeleteBackupRequest is synthesised, and the Velero Backup survives. // Without the branch the default falls through to cleanupVeleroBackup, diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index a61ec4a43..f9b9973ad 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,11 +60,22 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + lister, _, err := mariadbStorage(ctx, store, mdbBackup) if err != nil { return mariadbDump{}, err } - others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) +} + +// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, +// for a caller that reaches the bucket by another endpoint than the one +// mdbBackup names. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return mariadbDump{}, errors.New("the operator Backup does not store to S3") + } + others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go new file mode 100644 index 000000000..483ded502 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup" + + // Retry cadence while a dump cannot be deleted or a restore from the + // Backup is still running. + mariadbCleanupRetryInterval = 30 * time.Second +) + +// mariadbDumpCleanupTimeout bounds the object-store round trips of one +// cleanup attempt. The Backup controller has a single worker, so an +// unreachable endpoint stalls every other Backup for this long. A variable +// so tests can shorten it. +var mariadbDumpCleanupTimeout = 10 * time.Second + +// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then +// the operator Backup that wrote it. Every run of an application writes +// under one prefix, so the only key deleted is one proven to be this +// Backup's: the dump it records, or, for a Backup taken before dumps were +// recorded, the one findMariaDBDump attributes to its operator Backup. When +// no dump is proven the Backup is released with a Warning and nothing is +// deleted, the operator Backup included, since it is what any later +// attribution would start from. A failed delete holds the Backup and +// retries, as the Redis and Rabbitmq cleanups do. +func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + logger := getLogger(ctx) + if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" { + logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation) + return ctrl.Result{}, nil + } + + opName := backup.Spec.DriverMetadata[mariadbBackupNameKey] + if opName == "" { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil + } + + // The operator Restore reads the dump through the operator Backup, so + // neither may go while a restore from this Backup runs. + restore, err := r.activeRestoreOf(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", + "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + restore, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil + } + + op := &mariadbtypes.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", + fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil + } + return ctrl.Result{}, err + } + + s3 := op.Spec.Storage.S3 + if s3 == nil { + return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName), + "PVC and Volume storage is not reachable from the controller"), nil + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix) + + // A failed Job is Complete=True too, with reason JobFailed, and wrote no + // dump: whatever its window holds is another run's. + if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil || + cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil + } + + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name != "" { + if _, ok := parseMariaDBLogicalBackupFile(name); !ok { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil + } + } + + conn, err := r.mariadbCleanupConnection(ctx, op) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err)) + } + + s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) + defer cancel() + store := r.objectStore() + if name == "" { + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil + } + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, err) + } + name = dump.Name + // Recorded first: once the dump is gone a retry could not match it + // again, and could match a neighbour's instead. + base := backup.DeepCopy() + backup.Spec.DriverMetadata[mariadbDumpFileKey] = name + backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err) + } + } + + remover, err := store.remover(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + policy := metav1.DeletePropagationBackground + if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) + } + logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) + return ctrl.Result{}, nil +} + +// activeRestoreOf returns the name of a RestoreJob that restores from +// backup and has not finished, or "". +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { + list := &backupsv1alpha1.RestoreJobList{} + if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list RestoreJobs: %w", err) + } + for i := range list.Items { + rj := &list.Items[i] + if rj.Spec.BackupRef.Name != backup.Name { + continue + } + if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + return rj.Name, nil + } + } + return "", nil +} + +// mariadbCleanupConnection returns how to reach the bucket op wrote to. A +// dump on the platform bucket, which op names by its bucket and by the +// Secret the platform credentials are projected to, is reached as the +// platform reaches it now, so a Backup taken before the endpoint moved is +// still deleted. Any other bucket, or the platform bucket while the +// platform credentials cannot be read, is reached as op recorded it; a +// delete that fails there is handled by retryMariaDBCleanup like any other. +func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { + if r.CredentialsConfig.IsEnabled() { + conn, bucket, err := r.platformS3Connection(ctx) + switch { + case err != nil: + getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) + case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: + // A strategy can pin a private CA for the platform bucket; the + // platform credentials carry none. + if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil { + ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key) + if err != nil { + return s3Connection{}, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil + } + } + return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3) +} + +// platformS3Connection reads the platform's backup bucket from the Secret +// cozy-backups-creds is projected from, and its TLS from the scheme of +// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets +// tls.enabled. It reads the source rather than projecting it: projecting +// writes a Secret, which a terminating namespace refuses. +func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) { + cfg := r.CredentialsConfig + src := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing, + Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)} + } + return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)} + } + creds, err := parseSourceSecret(src) + if err != nil { + return s3Connection{}, "", err + } + if creds.endpoint == "" { + creds.endpoint = stripScheme(cfg.Endpoint) + } + if creds.endpoint == "" { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)} + } + if creds.region == "" { + creds.region = cfg.Region + } + return s3Connection{ + Endpoint: creds.endpoint, + Secure: strings.HasPrefix(cfg.Endpoint, "https://"), + Region: creds.region, + AccessKey: creds.accessKey, + SecretKey: creds.secretKey, + }, creds.bucket, nil +} + +// retryMariaDBCleanup holds the Backup after a failed attempt and says so, +// unless its namespace is going away, where nothing it needs can be read +// any longer and holding it would hold the namespace. +func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) { + terminating, err := r.namespaceTerminating(ctx, backup.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if terminating { + return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil + } + getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed", + "cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump", + location, cause, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil +} + +// releaseMariaDBCleanup lets the Backup go without deleting anything further +// and records a Warning naming what was left, so the give-up is visible. +func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result { + getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason) + } + return ctrl.Result{} +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go new file mode 100644 index 000000000..745f7aef4 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// fakeS3Bucket stands in for the object store on both the listing and the +// delete path, and records what was asked of it. +type fakeS3Bucket struct { + objects []s3ObjectInfo + listErr error + removeErr error + + conns []s3Connection + removed []string +} + +func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + return f.objects, f.listErr +} + +func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error { + if f.removeErr != nil { + return f.removeErr + } + f.removed = append(f.removed, bucket+"/"+key) + return nil +} + +var ( + cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + cleanupCompleted = cleanupCreated.Add(3 * time.Minute) + cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql" +) + +const cleanupPrefix = "tenant/db/" + +// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of: +// it stored to bkt under tenant/db with the tenant's creds Secret, and its +// Job succeeded. +func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup { + t.Helper() + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + op := operatorBackupStoringTo(t, s3, cleanupCreated) + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))} + return op +} + +// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup +// op-bk, recording dumpFile when it is not empty. +func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup { + apiGroup := strategyv1alpha1.GroupVersion.Group + md := map[string]string{ + mariadbBackupNameKey: "op-bk", + mariadbBackupNamespaceKey: "tenant", + } + if dumpFile != "" { + md[mariadbDumpFileKey] = dumpFile + } + now := metav1.Now() + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "bk", + DeletionTimestamp: &now, + Finalizers: []string{backupFinalizer}, + }, + Spec: backupsv1alpha1.BackupSpec{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb", + }, + TakenAt: metav1.NewTime(cleanupCompleted), + DriverMetadata: md, + }, + } +} + +func tenantNamespace() *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}} +} + +func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) { + t.Helper() + return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket) +} + +func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: rec, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + }, rec +} + +func operatorBackupExists(t *testing.T, c client.Client) bool { + t.Helper() + err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("get operator Backup: %v", err) + } + return err == nil +} + +func requeues(res ctrl.Result) bool { + return res.Requeue || res.RequeueAfter > 0 +} + +func hasEvent(events []string, prefix string) bool { + for _, e := range events { + if strings.HasPrefix(e, prefix) { + return true + } + } + return false +} + +// assertReleasedUntouched checks a Backup that is let go with nothing +// deleted: no S3 delete, the operator Backup kept, and a Warning saying so. +func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) { + t.Helper() + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 { + t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns) + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } +} + +func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil { + t.Fatalf("err = %v", err) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events) + } +} + +func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil { + t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held") + } +} + +// A legacy Backup records the dump it proved before deleting it, so a retry +// after a failed operator Backup delete finds the key by name instead of +// matching again against a bucket that no longer holds it. +func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + failDelete := true + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err == nil { + t.Fatal("first pass: err = nil, want the failed operator Backup delete returned") + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("Backup gone after a failed operator Backup delete: %v", err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName { + t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata) + } + + // The dump is gone now, as the first pass deleted it. + bucket.objects = nil + failDelete = false + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("second pass: %v", err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted on the retry") + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no claim that the deleted dump was left behind", events) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil { + t.Fatal("err = nil, want the failed record returned so the Backup is held") + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) { + for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + {Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}, + }} + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) { + t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata) + } +} + +// The window is the operator Backup's lifetime, closed by the Backup's +// takenAt: a dump stamped after takenAt plus the skew is not this run's. +func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) { + late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late}, + }} + op := cleanupOperatorBackup(t) + // The operator's own Complete transition is not what bounds the window. + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + cases := map[string][]s3ObjectInfo{ + "no dump in its lifetime": { + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + }, + "two dumps in its lifetime": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}, + {Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)}, + }, + "a dump placed by one clock only": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)}, + }, + } + for name, objects := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: objects} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}} + neighbour := cleanupOperatorBackup(t) + neighbour.Name = "op-neighbour" + neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second)) + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) { + t.Run("operator Backup is gone", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } + }) + t.Run("Backup records no operator Backup", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + delete(backup.Spec.DriverMetadata, mariadbBackupNameKey) + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup's Job failed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup has no Complete condition", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup never completed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Status = metav1.ConditionFalse + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("dump is on a volume", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage = mariadbtypes.BackupStorage{} + r, rec := cleanupReconciler(t, bucket, op, tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) +} + +func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"} + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob { + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}}, + Status: backupsv1alpha1.RestoreJobStatus{Phase: phase}, + } +} + +func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { + for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { + t.Run(string(phase), func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", phase)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events) + } + }) + } + t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded), + restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed), + restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) +} + +// platformCredsConfig points the reconciler at the platform's credentials +// Secret, the source the per-tenant Secret is projected from. The projected +// Secret is "creds", the one cleanupOperatorBackup names, as the default +// strategy names cozy-backups-creds. +func platformCredsConfig(endpoint string) BackupCredentialsConfig { + return BackupCredentialsConfig{ + SourceNamespace: "cozy-system", + SourceSecretName: "cozy-backups-credentials", + TargetSecretName: "creds", + Endpoint: endpoint, + Region: "us-east-1", + } +} + +func platformCredsSecret(bucketName, endpoint string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"}, + Data: map[string][]byte{ + "accessKey": []byte("PLATFORM"), + "secretKey": []byte("platform-secret"), + "endpoint": []byte(endpoint), + "bucketName": []byte(bucketName), + }, + } +} + +func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" { + t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn) + } + } +} + +func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("another-bucket", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn) + } + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + for _, bucketName := range []string{"bkt", "tenant-own-bucket"} { + t.Run(bucketName, func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.Bucket = bucketName + // No platform Secret: the projection source is missing. + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } + }) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}} + r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" { + t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("http://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed) + } +} + +func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("https://s3.configured.example") + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil { + t.Fatal(err) + } + if bucket.conns[0].Endpoint != "s3.configured.example" { + t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0]) + } +} + +// blockingBucket answers only when its context ends, as an unreachable +// endpoint does. +type blockingBucket struct{ fakeS3Bucket } + +func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { + <-ctx.Done() + return ctx.Err() +} + +func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { + prev := mariadbDumpCleanupTimeout + mariadbDumpCleanupTimeout = 50 * time.Millisecond + t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) + bucket := &blockingBucket{} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not give up on an unreachable endpoint") + } +} + +func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()} + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName()) + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } +} + +// A strategy of its own that happens to use the platform bucket's name, as +// an admin-chosen name on external S3 can, is reached as recorded: only the +// projected Secret marks a Backup as written to the platform bucket. +func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds" + op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds" + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } +} + +// With no endpoint anywhere the platform credentials are unusable, which +// falls back like any other unreadable credentials. +func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + malformed := platformCredsSecret("bkt", "s3.new.example") + delete(malformed.Data, "secretKey") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// Reconcile keeps the finalizer while the dump cannot be deleted, and drops +// it once both the dump and the operator Backup are gone. +func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503")} + backup := cleanupBackup(cleanupDumpName) + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil { + t.Fatalf("Backup gone while its dump could not be deleted: %v", err) + } + + bucket.removeErr = nil + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } +} + +// The minio adapter deletes the exact key, with the colons of the dump's +// timestamp escaped in the path; SeaweedFS refuses them raw. +func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) { + var method, rawPath, auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil { + t.Fatalf("RemoveObject: %v", err) + } + if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" { + t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath) + } + if !strings.Contains(auth, "Credential=AKID/") { + t.Errorf("request not signed with the access key: %q", auth) + } +} + +func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil { + t.Fatal("RemoveObject = nil, want the AccessDenied error") + } +}