From 55f1fd33a8dd8b73c196d4b45bb328e4c1791a74 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:04:04 +0300 Subject: [PATCH 01/18] fix(backups): let the controller delete operator MariaDB Backups Deleting a MariaDB Backup now deletes the k8s.mariadb.com/Backup that wrote its dump, once the dump itself is gone, so the controller needs the delete verb on that resource. Restores stay create-only. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../templates/rbac.yaml | 11 ++++---- .../tests/mariadb_cleanup_rbac_test.yaml | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 8d90980a8..e9e35b066 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -146,15 +146,16 @@ rules: # MariaDB strategy: read the operator-side MariaDB CR to surface "not yet # rendered" preconditions, create one-shot k8s.mariadb.com/Backup CRs on # the BackupJob path, and create k8s.mariadb.com/Restore CRs on the -# RestoreJob path. The driver only ever Creates/Lists/Gets these objects - -# it does not patch or delete them; their lifecycle is owned by -# mariadb-operator and (for cleanup) by tenants / explicit teardown flows, -# never by this controller. +# RestoreJob path. A Backup is deleted once the dump it wrote is, when its +# Cozystack Backup is deleted; a Restore is never deleted. - apiGroups: ["k8s.mariadb.com"] resources: ["mariadbs"] verbs: ["get", "list", "watch"] - apiGroups: ["k8s.mariadb.com"] - resources: ["backups", "restores"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] verbs: ["get", "list", "watch", "create"] # MongoDB strategy: read the operator-side PerconaServerMongoDB CR to gate on # the backup preconditions, create one-shot PerconaServerMongoDBBackup CRs on the diff --git a/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml new file mode 100644 index 000000000..21ac6e807 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml @@ -0,0 +1,28 @@ +suite: RBAC of the MariaDB dump cleanup + +# Deleting a MariaDB Backup deletes its operator Backup once the dump is gone. +# The Go unit tests run with no authorization, so a missing verb only shows up +# on a cluster, where every such Backup would stay Terminating. + +templates: + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "ClusterRole can delete operator Backups but not Restores" + asserts: + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] + verbs: ["get", "list", "watch", "create"] From f147b22d86343eb25aa8ff88651efc5ce7993a95 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:04:11 +0300 Subject: [PATCH 02/18] fix(backups): delete a MariaDB dump when its Backup is deleted Deleting a MariaDB Backup left its dump in the bucket for good: mariadb-operator's maxRetention prunes only from inside the next run of the same prefix, and a Plan never deletes a Backup, so on the platform bucket nothing else ever removes it. Every run of an application writes under one prefix, so the key deleted has to be proven to be this Backup's. It is the dump the Backup records, or, for a Backup taken before dumps were recorded, the one the existing ownership match attributes to its operator Backup. That match is recorded on the Backup before the delete, so a retry deletes the same key by name instead of matching again against a bucket that no longer holds it. Without proof the Backup is released with a Warning and nothing is deleted, the operator Backup included, since any later attribution starts from it. The operator Backup is deleted only after its dump, and the delete waits while a RestoreJob still restores from the Backup. The delete runs in the controller through the same minio client the size probe uses, so it verifies TLS the way mariadb-operator did when it wrote the dump. A dump on the platform bucket, which the operator Backup names by its bucket and by the projected credentials Secret, is reached through the platform's current endpoint and keys, still trusting a CA the strategy pinned, so a Backup taken before the endpoint moved is still deleted. Platform credentials that cannot be read fall back to the ones the operator Backup names. A failed delete holds the Backup and retries with a Warning naming the skip-artifact-cleanup annotation, except in a terminating namespace, where the Backup is released with a Warning so it does not hold the namespace. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/artifact_size.go | 38 +- .../backupcontroller/backup_controller.go | 33 +- .../backup_controller_test.go | 7 +- .../backupcontroller/mariadb_artifact_size.go | 15 +- .../mariadbstrategy_cleanup.go | 268 ++++++ .../mariadbstrategy_cleanup_test.go | 886 ++++++++++++++++++ 6 files changed, 1223 insertions(+), 24 deletions(-) create mode 100644 internal/backupcontroller/mariadbstrategy_cleanup.go create mode 100644 internal/backupcontroller/mariadbstrategy_cleanup_test.go diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 9197f07be..40d827c62 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -38,6 +38,11 @@ type s3ObjectLister interface { ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) } +type s3ObjectRemover interface { + // RemoveObject deletes key; a key that does not exist is not an error. + RemoveObject(ctx context.Context, bucket, key string) error +} + // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { @@ -63,8 +68,10 @@ type objectStore struct { // reader reads Secrets, which the manager's client fetches uncached, // and lists operator Backups, which it serves from the cache. reader client.Reader - // newLister overrides the S3 client; nil means a real minio client. - newLister func(conn s3Connection) (s3ObjectLister, error) + // newLister and newRemover override the S3 client; nil means a real + // minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) + newRemover func(conn s3Connection) (s3ObjectRemover, error) } func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { @@ -74,6 +81,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { return newMinioObjectLister(conn) } +func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) { + if s.newRemover != nil { + return s.newRemover(conn) + } + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { @@ -94,6 +112,10 @@ func (r *RestoreJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *BackupReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } @@ -103,6 +125,14 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + +func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) { opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -152,6 +182,10 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } +func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error { + return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}) +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..f817e696b 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -39,9 +39,16 @@ type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds - // into the Backup's namespace before spawning its best-effort delete Job. + // CredentialsConfig locates the platform backup credentials: the Job-based + // cleanups project them into the Backup's namespace, and the MariaDB + // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + + // newS3ObjectLister and newS3ObjectRemover override the S3 client the + // MariaDB cleanup deletes a dump with; nil means a real minio client. + // Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) + newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error) } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -126,22 +133,18 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: - // Cozystack Backup deletion does NOT delete the operator-side - // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. - // Lifecycle of that CR is owned by mariadb-operator's - // MaxRetention setting and by tenants / explicit teardown flows; - // the controller's RBAC has no delete verb on - // k8s.mariadb.com/backups for that reason. Surfacing this as an - // explicit branch (rather than falling through to the Velero - // default) keeps the contract honest: the MariaDB driver does - // not own the archive, so it does not delete it on CR removal. - // Mirrors the Altinity branch's "we do not own S3" stance. - return ctrl.Result{}, nil + // mariadb-operator's maxRetention prunes only from inside the next + // backup run of the same prefix and a Plan never deletes a Backup, so + // a dump on the platform bucket is deleted here or never. A dump in + // a tenant's own bucket is deleted too: the Backup proves it owns + // it, and the tenant deleting the Backup is asking for exactly that + // (see cleanupMariaDBBackup). + return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the - // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup + // archive" contract as Altinity / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer @@ -154,7 +157,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own - // the archive" contract as Altinity / MariaDB — the explicit + // the archive" contract as Altinity — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. diff --git a/internal/backupcontroller/backup_controller_test.go b/internal/backupcontroller/backup_controller_test.go index 579cb5fe6..d77f5d208 100644 --- a/internal/backupcontroller/backup_controller_test.go +++ b/internal/backupcontroller/backup_controller_test.go @@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) { // TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same // invariant as the Altinity test above for the MariaDB strategy. The -// dispatcher must explicitly route MariaDB-strategy Backups to a no-op -// cleanup rather than fall through to the Velero default - the MariaDB -// driver does not own the operator-side k8s.mariadb.com/Backup CR or the -// S3/PVC archive behind it (rbac.yaml grants no delete verb on -// k8s.mariadb.com/backups for that reason). The test seeds the Velero +// dispatcher must route MariaDB-strategy Backups to their own cleanup +// rather than fall through to the Velero default. The test seeds the Velero // metadata keys plus a matching velero.io/Backup; with the explicit branch // no DeleteBackupRequest is synthesised, and the Velero Backup survives. // Without the branch the default falls through to cleanupVeleroBackup, diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index a61ec4a43..f9b9973ad 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,11 +60,22 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + lister, _, err := mariadbStorage(ctx, store, mdbBackup) if err != nil { return mariadbDump{}, err } - others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) +} + +// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, +// for a caller that reaches the bucket by another endpoint than the one +// mdbBackup names. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return mariadbDump{}, errors.New("the operator Backup does not store to S3") + } + others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go new file mode 100644 index 000000000..483ded502 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup" + + // Retry cadence while a dump cannot be deleted or a restore from the + // Backup is still running. + mariadbCleanupRetryInterval = 30 * time.Second +) + +// mariadbDumpCleanupTimeout bounds the object-store round trips of one +// cleanup attempt. The Backup controller has a single worker, so an +// unreachable endpoint stalls every other Backup for this long. A variable +// so tests can shorten it. +var mariadbDumpCleanupTimeout = 10 * time.Second + +// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then +// the operator Backup that wrote it. Every run of an application writes +// under one prefix, so the only key deleted is one proven to be this +// Backup's: the dump it records, or, for a Backup taken before dumps were +// recorded, the one findMariaDBDump attributes to its operator Backup. When +// no dump is proven the Backup is released with a Warning and nothing is +// deleted, the operator Backup included, since it is what any later +// attribution would start from. A failed delete holds the Backup and +// retries, as the Redis and Rabbitmq cleanups do. +func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + logger := getLogger(ctx) + if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" { + logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation) + return ctrl.Result{}, nil + } + + opName := backup.Spec.DriverMetadata[mariadbBackupNameKey] + if opName == "" { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil + } + + // The operator Restore reads the dump through the operator Backup, so + // neither may go while a restore from this Backup runs. + restore, err := r.activeRestoreOf(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", + "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + restore, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil + } + + op := &mariadbtypes.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", + fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil + } + return ctrl.Result{}, err + } + + s3 := op.Spec.Storage.S3 + if s3 == nil { + return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName), + "PVC and Volume storage is not reachable from the controller"), nil + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix) + + // A failed Job is Complete=True too, with reason JobFailed, and wrote no + // dump: whatever its window holds is another run's. + if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil || + cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil + } + + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name != "" { + if _, ok := parseMariaDBLogicalBackupFile(name); !ok { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil + } + } + + conn, err := r.mariadbCleanupConnection(ctx, op) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err)) + } + + s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) + defer cancel() + store := r.objectStore() + if name == "" { + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil + } + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, err) + } + name = dump.Name + // Recorded first: once the dump is gone a retry could not match it + // again, and could match a neighbour's instead. + base := backup.DeepCopy() + backup.Spec.DriverMetadata[mariadbDumpFileKey] = name + backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err) + } + } + + remover, err := store.remover(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + policy := metav1.DeletePropagationBackground + if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) + } + logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) + return ctrl.Result{}, nil +} + +// activeRestoreOf returns the name of a RestoreJob that restores from +// backup and has not finished, or "". +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { + list := &backupsv1alpha1.RestoreJobList{} + if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list RestoreJobs: %w", err) + } + for i := range list.Items { + rj := &list.Items[i] + if rj.Spec.BackupRef.Name != backup.Name { + continue + } + if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + return rj.Name, nil + } + } + return "", nil +} + +// mariadbCleanupConnection returns how to reach the bucket op wrote to. A +// dump on the platform bucket, which op names by its bucket and by the +// Secret the platform credentials are projected to, is reached as the +// platform reaches it now, so a Backup taken before the endpoint moved is +// still deleted. Any other bucket, or the platform bucket while the +// platform credentials cannot be read, is reached as op recorded it; a +// delete that fails there is handled by retryMariaDBCleanup like any other. +func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { + if r.CredentialsConfig.IsEnabled() { + conn, bucket, err := r.platformS3Connection(ctx) + switch { + case err != nil: + getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) + case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: + // A strategy can pin a private CA for the platform bucket; the + // platform credentials carry none. + if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil { + ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key) + if err != nil { + return s3Connection{}, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil + } + } + return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3) +} + +// platformS3Connection reads the platform's backup bucket from the Secret +// cozy-backups-creds is projected from, and its TLS from the scheme of +// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets +// tls.enabled. It reads the source rather than projecting it: projecting +// writes a Secret, which a terminating namespace refuses. +func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) { + cfg := r.CredentialsConfig + src := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing, + Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)} + } + return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)} + } + creds, err := parseSourceSecret(src) + if err != nil { + return s3Connection{}, "", err + } + if creds.endpoint == "" { + creds.endpoint = stripScheme(cfg.Endpoint) + } + if creds.endpoint == "" { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)} + } + if creds.region == "" { + creds.region = cfg.Region + } + return s3Connection{ + Endpoint: creds.endpoint, + Secure: strings.HasPrefix(cfg.Endpoint, "https://"), + Region: creds.region, + AccessKey: creds.accessKey, + SecretKey: creds.secretKey, + }, creds.bucket, nil +} + +// retryMariaDBCleanup holds the Backup after a failed attempt and says so, +// unless its namespace is going away, where nothing it needs can be read +// any longer and holding it would hold the namespace. +func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) { + terminating, err := r.namespaceTerminating(ctx, backup.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if terminating { + return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil + } + getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed", + "cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump", + location, cause, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil +} + +// releaseMariaDBCleanup lets the Backup go without deleting anything further +// and records a Warning naming what was left, so the give-up is visible. +func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result { + getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason) + } + return ctrl.Result{} +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go new file mode 100644 index 000000000..745f7aef4 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// fakeS3Bucket stands in for the object store on both the listing and the +// delete path, and records what was asked of it. +type fakeS3Bucket struct { + objects []s3ObjectInfo + listErr error + removeErr error + + conns []s3Connection + removed []string +} + +func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + return f.objects, f.listErr +} + +func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error { + if f.removeErr != nil { + return f.removeErr + } + f.removed = append(f.removed, bucket+"/"+key) + return nil +} + +var ( + cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + cleanupCompleted = cleanupCreated.Add(3 * time.Minute) + cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql" +) + +const cleanupPrefix = "tenant/db/" + +// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of: +// it stored to bkt under tenant/db with the tenant's creds Secret, and its +// Job succeeded. +func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup { + t.Helper() + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + op := operatorBackupStoringTo(t, s3, cleanupCreated) + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))} + return op +} + +// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup +// op-bk, recording dumpFile when it is not empty. +func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup { + apiGroup := strategyv1alpha1.GroupVersion.Group + md := map[string]string{ + mariadbBackupNameKey: "op-bk", + mariadbBackupNamespaceKey: "tenant", + } + if dumpFile != "" { + md[mariadbDumpFileKey] = dumpFile + } + now := metav1.Now() + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "bk", + DeletionTimestamp: &now, + Finalizers: []string{backupFinalizer}, + }, + Spec: backupsv1alpha1.BackupSpec{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb", + }, + TakenAt: metav1.NewTime(cleanupCompleted), + DriverMetadata: md, + }, + } +} + +func tenantNamespace() *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}} +} + +func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) { + t.Helper() + return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket) +} + +func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: rec, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + }, rec +} + +func operatorBackupExists(t *testing.T, c client.Client) bool { + t.Helper() + err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("get operator Backup: %v", err) + } + return err == nil +} + +func requeues(res ctrl.Result) bool { + return res.Requeue || res.RequeueAfter > 0 +} + +func hasEvent(events []string, prefix string) bool { + for _, e := range events { + if strings.HasPrefix(e, prefix) { + return true + } + } + return false +} + +// assertReleasedUntouched checks a Backup that is let go with nothing +// deleted: no S3 delete, the operator Backup kept, and a Warning saying so. +func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) { + t.Helper() + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 { + t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns) + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } +} + +func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil { + t.Fatalf("err = %v", err) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events) + } +} + +func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil { + t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held") + } +} + +// A legacy Backup records the dump it proved before deleting it, so a retry +// after a failed operator Backup delete finds the key by name instead of +// matching again against a bucket that no longer holds it. +func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + failDelete := true + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err == nil { + t.Fatal("first pass: err = nil, want the failed operator Backup delete returned") + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("Backup gone after a failed operator Backup delete: %v", err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName { + t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata) + } + + // The dump is gone now, as the first pass deleted it. + bucket.objects = nil + failDelete = false + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("second pass: %v", err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted on the retry") + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no claim that the deleted dump was left behind", events) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil { + t.Fatal("err = nil, want the failed record returned so the Backup is held") + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) { + for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + {Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}, + }} + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) { + t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata) + } +} + +// The window is the operator Backup's lifetime, closed by the Backup's +// takenAt: a dump stamped after takenAt plus the skew is not this run's. +func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) { + late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late}, + }} + op := cleanupOperatorBackup(t) + // The operator's own Complete transition is not what bounds the window. + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + cases := map[string][]s3ObjectInfo{ + "no dump in its lifetime": { + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + }, + "two dumps in its lifetime": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}, + {Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)}, + }, + "a dump placed by one clock only": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)}, + }, + } + for name, objects := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: objects} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}} + neighbour := cleanupOperatorBackup(t) + neighbour.Name = "op-neighbour" + neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second)) + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) { + t.Run("operator Backup is gone", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } + }) + t.Run("Backup records no operator Backup", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + delete(backup.Spec.DriverMetadata, mariadbBackupNameKey) + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup's Job failed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup has no Complete condition", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup never completed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Status = metav1.ConditionFalse + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("dump is on a volume", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage = mariadbtypes.BackupStorage{} + r, rec := cleanupReconciler(t, bucket, op, tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) +} + +func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"} + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob { + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}}, + Status: backupsv1alpha1.RestoreJobStatus{Phase: phase}, + } +} + +func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { + for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { + t.Run(string(phase), func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", phase)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events) + } + }) + } + t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded), + restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed), + restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) +} + +// platformCredsConfig points the reconciler at the platform's credentials +// Secret, the source the per-tenant Secret is projected from. The projected +// Secret is "creds", the one cleanupOperatorBackup names, as the default +// strategy names cozy-backups-creds. +func platformCredsConfig(endpoint string) BackupCredentialsConfig { + return BackupCredentialsConfig{ + SourceNamespace: "cozy-system", + SourceSecretName: "cozy-backups-credentials", + TargetSecretName: "creds", + Endpoint: endpoint, + Region: "us-east-1", + } +} + +func platformCredsSecret(bucketName, endpoint string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"}, + Data: map[string][]byte{ + "accessKey": []byte("PLATFORM"), + "secretKey": []byte("platform-secret"), + "endpoint": []byte(endpoint), + "bucketName": []byte(bucketName), + }, + } +} + +func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" { + t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn) + } + } +} + +func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("another-bucket", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn) + } + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + for _, bucketName := range []string{"bkt", "tenant-own-bucket"} { + t.Run(bucketName, func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.Bucket = bucketName + // No platform Secret: the projection source is missing. + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } + }) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}} + r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" { + t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("http://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed) + } +} + +func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("https://s3.configured.example") + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil { + t.Fatal(err) + } + if bucket.conns[0].Endpoint != "s3.configured.example" { + t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0]) + } +} + +// blockingBucket answers only when its context ends, as an unreachable +// endpoint does. +type blockingBucket struct{ fakeS3Bucket } + +func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { + <-ctx.Done() + return ctx.Err() +} + +func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { + prev := mariadbDumpCleanupTimeout + mariadbDumpCleanupTimeout = 50 * time.Millisecond + t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) + bucket := &blockingBucket{} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not give up on an unreachable endpoint") + } +} + +func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()} + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName()) + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } +} + +// A strategy of its own that happens to use the platform bucket's name, as +// an admin-chosen name on external S3 can, is reached as recorded: only the +// projected Secret marks a Backup as written to the platform bucket. +func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds" + op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds" + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } +} + +// With no endpoint anywhere the platform credentials are unusable, which +// falls back like any other unreadable credentials. +func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + malformed := platformCredsSecret("bkt", "s3.new.example") + delete(malformed.Data, "secretKey") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// Reconcile keeps the finalizer while the dump cannot be deleted, and drops +// it once both the dump and the operator Backup are gone. +func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503")} + backup := cleanupBackup(cleanupDumpName) + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil { + t.Fatalf("Backup gone while its dump could not be deleted: %v", err) + } + + bucket.removeErr = nil + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } +} + +// The minio adapter deletes the exact key, with the colons of the dump's +// timestamp escaped in the path; SeaweedFS refuses them raw. +func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) { + var method, rawPath, auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil { + t.Fatalf("RemoveObject: %v", err) + } + if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" { + t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath) + } + if !strings.Contains(auth, "Credential=AKID/") { + t.Errorf("request not signed with the access key: %q", auth) + } +} + +func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil { + t.Fatal("RemoveObject = nil, want the AccessDenied error") + } +} From e66b764734a505a85009922c36629a4d674512c6 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:21:22 +0300 Subject: [PATCH 03/18] fix(backups): let an unfinished MariaDB run block a legacy dump delete A Backup taken before dumps were recorded finds its dump by window, and only finished neighbour runs were allowed to contest a candidate. A run uploads its dump and prunes older ones through maxRetention before its operator Backup turns Complete, so in that gap the pruned run's cleanup saw the newer run's dump as the only candidate and deleted it, leaving that Backup Ready with nothing to restore. The size probe and the restore keep ignoring unfinished runs, where a wrong guess costs a size or a refused restore. Deletion now treats a run that has not completed as one whose window is still open, so the dump is left behind with a Warning, as for any other unproven attribution. Holding the Backup would not help: once that run completes its window still covers the candidate, and the attribution stays ambiguous. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/mariadb_artifact_size.go | 37 ++++++++----- .../mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 52 +++++++++++++++++++ 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index f9b9973ad..4e0525517 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -64,18 +64,19 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt if err != nil { return mariadbDump{}, err } - return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt, false) } // findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, // for a caller that reaches the bucket by another endpoint than the one -// mdbBackup names. -func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { +// mdbBackup names. withUnfinished lets runs that have not completed contest +// the dump too, as neighbourMariaDBRuns describes. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time, withUnfinished bool) (mariadbDump, error) { s3 := mdbBackup.Spec.Storage.S3 if s3 == nil { return mariadbDump{}, errors.New("the operator Backup does not store to S3") } - others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) + others, err := neighbourMariaDBRuns(ctx, reader, mdbBackup, withUnfinished) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } @@ -136,6 +137,10 @@ type mariadbRun struct { lower, upper time.Time } +// mariadbUnfinishedRunEnd closes the window of a run that has not completed, +// which can still write its dump at any time. +var mariadbUnfinishedRunEnd = time.Date(9999, time.December, 31, 0, 0, 0, 0, time.UTC) + func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { return mariadbRun{ name: b.Namespace + "/" + b.Name, @@ -161,8 +166,8 @@ func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { // dump in the window on one clock only may be this run's own, misplaced by // a skewed clock, which leaves any other candidate unproven. A neighbour's // dump is still taken if this run's own is missing from the listing, or -// misplaced on both clocks, and the neighbour's operator Backup is gone or -// has not completed. +// misplaced on both clocks, and the neighbour's operator Backup is gone or, +// unless others includes unfinished runs, has not completed. func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { var near []mariadbDump for _, obj := range objects { @@ -194,14 +199,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, oth return d, nil } -// finishedMariaDBRuns returns the runs of the other operator Backups that +// neighbourMariaDBRuns returns the runs of the other operator Backups that // write under the same bucket and prefix as b and have completed, whether -// their Job succeeded or failed. One still running is left out: its window -// has no end yet, and an operator Backup stuck short of Complete, as one -// whose Job pod never starts is, would otherwise contest every later run of -// the application for good. The endpoint is not compared, since one has -// several spellings, and a false neighbour only leaves a size unset. -func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { +// their Job succeeded or failed. One still running is left out unless +// withUnfinished is set: its window has no end yet, and an operator Backup +// stuck short of Complete, as one whose Job pod never starts is, would +// otherwise contest every later run of the application for good. That is +// the price of deleting a dump, since a run can upload its dump and prune +// older ones through maxRetention before it completes. The endpoint is not +// compared, since one has several spellings, and a false neighbour only +// keeps a dump unattributed. +func neighbourMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup, withUnfinished bool) ([]mariadbRun, error) { list := &mariadbtypes.BackupList{} if err := reader.List(ctx, list); err != nil { return nil, err @@ -218,6 +226,9 @@ func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbty } cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) if cond == nil || cond.Status != metav1.ConditionTrue { + if withUnfinished { + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, mariadbUnfinishedRunEnd)) + } continue } runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 483ded502..0a2fc1e34 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -117,7 +117,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err != nil { return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) } - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 745f7aef4..9b41f4701 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -385,6 +385,58 @@ func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased( assertReleasedUntouched(t, r, rec, bucket, res, err) } +// A run started before this one completed can upload its dump and prune +// this run's through maxRetention before its own operator Backup completes. +// Its dump is then the only one in this run's window, and only its open +// window says it may be its own. +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpAnUnfinishedRunCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(2 * time.Minute) + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, started.Add(90*time.Second), "gzip"), LastModified: started.Add(100 * time.Second)}, + }} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(started) + running.Status.Conditions = conditions + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// An unfinished run contests only what it could have written: a dump +// that predates it, on both clocks, is still this run's. +func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + backup := cleanupBackup("") + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 0972690f0d31426f892e7d7e04ad48cf33d5e9c3 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 13:52:38 +0300 Subject: [PATCH 04/18] fix(backups): contest a recorded MariaDB dump before deleting it The size probe records the dump it attributes to a Backup, and it lets only runs that had already completed contest the attribution. A run can upload its dump and prune an older run's through maxRetention before its own operator Backup completes, so the probe of the pruned run can record the newer run's dump, and deleting that Backup then deleted a dump that another Backup still restores from. Deletion now contests the recorded dump against every run on the same bucket and prefix as they stand at deletion time, the unfinished ones included, and leaves it behind with a Warning when any could have written it, as the unrecorded path already does. A run that never completes therefore keeps every later dump its lifetime covers, which leaks storage rather than losing a backup. The probe is unchanged here. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 53 ++++- .../mariadbstrategy_cleanup_test.go | 215 ++++++++++++++++++ 2 files changed, 264 insertions(+), 4 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 0a2fc1e34..94a7d10f4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "time" @@ -37,7 +38,8 @@ var mariadbDumpCleanupTimeout = 10 * time.Second // cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then // the operator Backup that wrote it. Every run of an application writes // under one prefix, so the only key deleted is one proven to be this -// Backup's: the dump it records, or, for a Backup taken before dumps were +// Backup's: the dump it records, unless contestedMariaDBDump finds another +// run that could have written it, or, for a Backup taken before dumps were // recorded, the one findMariaDBDump attributes to its operator Backup. When // no dump is proven the Backup is released with a Warning and nothing is // deleted, the operator Backup included, since it is what any later @@ -112,11 +114,20 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) defer cancel() store := r.objectStore() - if name == "" { - lister, err := store.lister(conn) + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if name != "" { + other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) if err != nil { - return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + return r.retryMariaDBCleanup(ctx, backup, location+name, err) } + if other != "" { + return r.releaseMariaDBCleanup(ctx, backup, location+name, + fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil + } + } else { dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, @@ -151,6 +162,40 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac return ctrl.Result{}, nil } +// contestedMariaDBDump returns the name of another operator Backup on op's +// bucket and prefix whose run, finished or not, could have written the dump +// name, or "" when none could. The size probe that recorded name let only +// runs finished by then contest it, and a run can upload its dump and prune +// op's through maxRetention before it completes. A dump no longer in the +// bucket is contested by nobody: deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) { + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + } + s3 := op.Spec.Storage.S3 + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) + if i < 0 { + return "", nil + } + others, err := neighbourMariaDBRuns(ctx, reader, op, true) + if err != nil { + return "", fmt.Errorf("list operator Backups: %w", err) + } + d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} + for _, other := range others { + if other.placesOnEitherClock(d) { + return other.name, nil + } + } + return "", nil +} + // activeRestoreOf returns the name of a RestoreJob that restores from // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 9b41f4701..ddca34523 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -22,6 +22,7 @@ import ( strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) @@ -437,6 +438,220 @@ func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedR } } +// The size probe that records a dump lets only finished runs contest it, so +// a run still unfinished then, which uploaded its dump and pruned this +// run's through maxRetention, can have its dump recorded here. Deletion +// contests the recorded dump again, with every run as it stands now. +func TestCleanupMariaDBBackup_RecordedDumpARunUnfinishedAtTheProbeCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + "completed since": {mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted.Add(5 * time.Minute)))}, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(started) + other.Status.Conditions = conditions + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// Either clock placing the recorded dump in another run's lifetime is +// enough: the backup pod's clock can be skewed while the store's is not. +func TestCleanupMariaDBBackup_RecordedDumpAnotherRunPlacesOnOneClockIsReleased(t *testing.T) { + stamped := cleanupCreated.Add(20 * time.Second) + uploaded := cleanupCreated.Add(2 * time.Minute) + theirs := dumpKey(cleanupPrefix, stamped, "gzip") + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: uploaded}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(uploaded.Add(30 * time.Second)) + other.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// The listing is by prefix, so a longer key sharing it is not the dump and +// contests nothing. +func TestCleanupMariaDBBackup_RecordedDumpIgnoresAKeyItIsOnlyAPrefixOf(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key + ".tmp", LastModified: cleanupCompleted.Add(2 * time.Hour)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// A name the dump time cannot be read from leaves one clock out of the +// contest, so it is an error rather than a contest on half the evidence. +func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { + op := cleanupOperatorBackup(t) + name := "backup.not-a-time.sql" + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} + c := newMariaDBStrategyTestClient(t, op) + if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + t.Fatal("err = nil, want the unreadable dump time reported") + } +} + +// A retry after the dump was deleted but the operator Backup was not finds +// nothing to contest, whichever run's lifetime the dump's name falls in. +func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T) { + bucket := &fakeS3Bucket{} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCreated) + running.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted") + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no ArtifactNotDeleted", events) + } +} + +// A run started after the recorded dump was uploaded cannot have written it. +func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } +} + +func TestCleanupMariaDBBackup_RecordedDumpListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// Without the list of runs nothing is known about who else could have +// written the dump, so it is kept and the delete retried. +func TestCleanupMariaDBBackup_RecordedDumpWithUnreadableRunsRetries(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.BackupList); ok { + return errors.New("apiserver unavailable") + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// The whole sequence: run B uploads its dump and prunes this run's before +// B completes, this run's Backup is created and probed, then deleted. B's +// dump must survive whatever the probe recorded. +func TestMariaDBBackup_ProbeThenCleanupKeepsADumpAnUnfinishedRunUploaded(t *testing.T) { + apps := mariadbapp.GroupName + started := cleanupCreated.Add(time.Minute) + theirs := s3ObjectInfo{Key: dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip"), Size: 42, LastModified: started.Add(time.Minute)} + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "tenant/db" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "db", APIGroup: &apps}, + }, + } + other := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-other" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + + prober := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{theirs}}, job, other(), s3CredsSecret("tenant", "creds")) + created, err := prober.createMariaDBBackupArtifact(context.Background(), job, resolved, cleanupOperatorBackup(t), rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if got, want := created.Spec.DriverMetadata[mariadbDumpFileKey], strings.TrimPrefix(theirs.Key, cleanupPrefix); got != want { + t.Fatalf("probe recorded dump %q, want %q: this test covers the recorded path", got, want) + } + + backup := created.DeepCopy() + backup.ResourceVersion = "" + now := metav1.Now() + backup.DeletionTimestamp = &now + backup.Finalizers = []string{backupFinalizer} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{theirs}} + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), other(), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 74fbb34aca35beabe6c42a7c02f89fd3bc7a40e8 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 18:47:08 +0300 Subject: [PATCH 05/18] docs(backups): say who deletes a MariaDB Backup The comment justified deleting a dump in a tenant's own bucket by the tenant asking for it, but tenant roles grant no delete on Backups: only an administrator or the namespace teardown deletes one, and the reader of this branch should know whose request the delete answers. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index f817e696b..9d755059e 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -137,8 +137,8 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // backup run of the same prefix and a Plan never deletes a Backup, so // a dump on the platform bucket is deleted here or never. A dump in // a tenant's own bucket is deleted too: the Backup proves it owns - // it, and the tenant deleting the Backup is asking for exactly that - // (see cleanupMariaDBBackup). + // it, and tenants cannot delete Backups, so the delete comes from an + // administrator or from namespace teardown (see cleanupMariaDBBackup). return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the From 320d0299ad8e4ebb4d34568fdc561473f2808b86 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:03:59 +0300 Subject: [PATCH 06/18] fix(backups): refuse a MariaDB restore from a Backup being deleted Deleting a MariaDB Backup now deletes its dump, and mariadb-operator restores the dump nearest the requested time when the exact one is gone. A restore that starts while the Backup's cleanup runs can lose its dump between the check and the operator's read, and then completes on another run's data without any error. The restore now reads the Backup from the API server before creating the operator Restore and fails when it is being deleted or is gone. The copy the reconcile was handed comes from the cache and can predate the delete; a transient read error keeps the RestoreJob waiting like any other failed dump check. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 52 +++++++++++++++++++ .../mariadbstrategy_controller.go | 14 +++++ 2 files changed, 66 insertions(+) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 6c8202adf..218c91de6 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" @@ -184,6 +185,57 @@ func recordedDump(name string) map[string]string { return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} } +// A Backup being deleted is deleting its dump, so a restore that starts +// now can find it gone once the operator reads it, and the operator then +// restores the nearest other dump. The Backup is read from the API server, +// since the copy the reconcile was handed can predate the delete. +func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + present := func() *fakeS3Lister { + return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}} + } + cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil} + for name, md := range cases { + t.Run(name+", being deleted", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "being deleted") + }) + t.Run(name+", already gone", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "is gone") + }) + } + t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable") + }) +} + func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { creds := s3CredsSecret("tenant", "creds") diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 6d6312eb4..e3e00dbec 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -669,6 +669,20 @@ func (e mariadbRestoreRefused) Error() string { return string(e) } // checked to still exist. A Backup that records no dump is resolved by // legacyMariaDBRestoreTargetTime. func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // Deleting a Backup deletes its dump, and the operator restores the + // nearest other dump once its own is gone, so a restore must not start + // from a Backup being deleted. The copy in hand can predate the delete; + // the cleanup holds the Backup only for RestoreJobs it can already see. + live := &backupsv1alpha1.Backup{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { + if apierrors.IsNotFound(err) { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name)) + } + return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err) + } + if !live.DeletionTimestamp.IsZero() { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name)) + } // A failed Job is Complete=True too, with reason JobFailed, and a // BackupJob that took Complete=True for success made a Backup of it. if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { From 654c543264bbcba9c939a7c8b436ffbc02340c4d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:04:53 +0300 Subject: [PATCH 07/18] fix(backups): see a MariaDB restore the cache has not caught up with A MariaDB restore now checks on the API server that its Backup is not being deleted before it starts, so a restore that started first exists on the API server before the delete. The cleanup read RestoreJobs from the informer cache, which can lag behind, and could delete the dump under a restore it had not seen yet. Reading them uncached closes that window from both sides. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 15 +++++++++++++++ .../backupcontroller/mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 9d755059e..d8c69bf1c 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -43,6 +43,10 @@ type BackupReconciler struct { // cleanups project them into the Backup's namespace, and the MariaDB // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + // APIReader is the manager's uncached reader. The MariaDB cleanup lists + // the RestoreJobs of a Backup through it: a restore that started before + // the delete may not be in the cache yet. Wired in SetupWithManager. + APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the // MariaDB cleanup deletes a dump with; nil means a real minio client. @@ -294,7 +298,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.APIReader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) } + +// apiReader returns the uncached reader. Reconcilers built by hand in unit +// tests have none and fall back to their fake Client, which has no cache to +// bypass. +func (r *BackupReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 94a7d10f4..531b806c9 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -200,7 +200,7 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { list := &backupsv1alpha1.RestoreJobList{} - if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) } for i := range list.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ddca34523..20f2757d4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -763,6 +763,18 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { } }) } + // A restore checks that its Backup is not being deleted before it + // starts, so one that started first is on the API server before the + // delete, though not necessarily in this controller's cache yet. + t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), From 37a74e59394a69700e99b04a697f50b9a78e2625 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:05:51 +0300 Subject: [PATCH 08/18] test(backups): reach the S3 calls the MariaDB cleanup timeout bounds Contesting a recorded dump lists the bucket before the delete, and the timeout test supplied no lister, so a real client failed on DNS before the blocking delete was ever called and the suite stayed green with the timeout removed. The test now lists the dump through the fake bucket, blocks in the listing and in the delete in turn, and requires that the blocking call was entered. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup_test.go | 75 +++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 20f2757d4..0ce6b2d01 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -961,36 +961,69 @@ func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *tes } } -// blockingBucket answers only when its context ends, as an unreachable -// endpoint does. -type blockingBucket struct{ fakeS3Bucket } +// blockingBucket lists the dump it holds and answers the call named by +// blockOn only when its context ends, as an unreachable endpoint does. +type blockingBucket struct { + fakeS3Bucket + blockOn string + entered chan struct{} +} -func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { +func (b *blockingBucket) block(ctx context.Context, call string) bool { + if b.blockOn != call { + return false + } + close(b.entered) <-ctx.Done() - return ctx.Err() + return true +} + +func (b *blockingBucket) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + if b.block(ctx, "list") { + return nil, ctx.Err() + } + return b.fakeS3Bucket.ListObjects(ctx, bucket, prefix, recursive) +} + +func (b *blockingBucket) RemoveObject(ctx context.Context, bucket, key string) error { + if b.block(ctx, "remove") { + return ctx.Err() + } + return b.fakeS3Bucket.RemoveObject(ctx, bucket, key) } func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { prev := mariadbDumpCleanupTimeout mariadbDumpCleanupTimeout = 50 * time.Millisecond t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) - bucket := &blockingBucket{} - c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) - r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), - newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + for _, call := range []string{"list", "remove"} { + t.Run(call, func(t *testing.T) { + bucket := &blockingBucket{blockOn: call, entered: make(chan struct{})} + bucket.objects = []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectLister: func(s3Connection) (s3ObjectLister, error) { return bucket, nil }, + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} - done := make(chan ctrl.Result, 1) - go func() { - res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) - done <- res - }() - select { - case res := <-done: - if !requeues(res) { - t.Errorf("res = %+v, want the Backup held and retried", res) - } - case <-time.After(5 * time.Second): - t.Fatal("cleanup did not give up on an unreachable endpoint") + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + select { + case <-bucket.entered: + default: + t.Fatalf("cleanup returned without reaching the blocking %s", call) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatalf("cleanup did not give up on an unreachable endpoint blocking %s", call) + } + }) } } From bbeca3453534124975682bda008f0d025aaed65c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:07:03 +0300 Subject: [PATCH 09/18] fix(backups): retry a failed read of the platform backup credentials The MariaDB cleanup reaches the platform bucket through the platform's current endpoint so that a Backup taken before the endpoint moved is still deleted. A failed read of the platform credentials made it fall back to the endpoint recorded on the operator Backup, which may be the old address: there the dump is not found, a missing dump counts as already deleted, and the operator Backup that pointed at the real dump was deleted with a success log. A read that fails with an API error now holds the Backup and retries. A missing or malformed platform Secret still falls back, since it is a standing state that a retry would not change. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 10 +++++-- .../mariadbstrategy_cleanup_test.go | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 531b806c9..b4617918e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -220,12 +220,18 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv // Secret the platform credentials are projected to, is reached as the // platform reaches it now, so a Backup taken before the endpoint moved is // still deleted. Any other bucket, or the platform bucket while the -// platform credentials cannot be read, is reached as op recorded it; a -// delete that fails there is handled by retryMariaDBCleanup like any other. +// platform credentials are missing or malformed, is reached as op recorded +// it; a delete that fails there is handled by retryMariaDBCleanup like any +// other. A failed read of the platform credentials is retried instead: the +// recorded endpoint may be the one the platform moved off, where the dump +// would be found missing and reported deleted. func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { if r.CredentialsConfig.IsEnabled() { conn, bucket, err := r.platformS3Connection(ctx) + var projErr *ProjectionError switch { + case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError: + return s3Connection{}, err case err != nil: getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 0ce6b2d01..1eecfeaff 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -869,6 +869,36 @@ func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperator } } +// A missing or malformed platform Secret is a standing state, and the +// operator Backup's own credentials are the best left. A failed read says +// nothing about the Secret: falling back then can list a moved endpoint's +// old address, find no dump there and report it deleted. +func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 3334152da7d78e755ab7d3cc85ed8d459c0ba86e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:08:43 +0300 Subject: [PATCH 10/18] fix(backups): report a recorded MariaDB dump that was not found A recorded dump missing from the listing was logged as deleted, with no event. That is right when an earlier attempt or maxRetention removed it, but the listing goes through the platform's current endpoint, and after an endpoint move the dump can still sit on the old storage: the cleanup then deleted the operator Backup that pointed at it and said it had deleted the dump. The cleanup now records a Normal ArtifactAlreadyAbsent event naming the key and the endpoint it listed, so whoever later looks for that dump can tell where it was not found. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 36 ++++++++++++------- .../mariadbstrategy_cleanup_test.go | 14 ++++++-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index b4617918e..25775185b 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -118,8 +118,10 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err != nil { return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) } + absent := false if name != "" { - other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) } @@ -158,42 +160,52 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) } + if absent { + logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent", + "%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted", + location+name, conn.Endpoint, opName) + } + return ctrl.Result{}, nil + } logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) return ctrl.Result{}, nil } // contestedMariaDBDump returns the name of another operator Backup on op's // bucket and prefix whose run, finished or not, could have written the dump -// name, or "" when none could. The size probe that recorded name let only -// runs finished by then contest it, and a run can upload its dump and prune -// op's through maxRetention before it completes. A dump no longer in the -// bucket is contested by nobody: deleting it again removes nothing. -func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) { +// name, or "" when none could, and whether the listing found the dump. The +// size probe that recorded name let only runs finished by then contest it, +// and a run can upload its dump and prune op's through maxRetention before +// it completes. A dump no longer in the bucket is contested by nobody: +// deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) { ts, ok := parseMariaDBLogicalBackupFile(name) if !ok { - return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name) } s3 := op.Spec.Storage.S3 key := mariadbOperatorS3Prefix(s3.Prefix) + name objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) if err != nil { - return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) } i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) if i < 0 { - return "", nil + return "", false, nil } others, err := neighbourMariaDBRuns(ctx, reader, op, true) if err != nil { - return "", fmt.Errorf("list operator Backups: %w", err) + return "", true, fmt.Errorf("list operator Backups: %w", err) } d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} for _, other := range others { if other.placesOnEitherClock(d) { - return other.name, nil + return other.name, true, nil } } - return "", nil + return "", true, nil } // activeRestoreOf returns the name of a RestoreJob that restores from diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 1eecfeaff..da9e0902e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -165,7 +165,7 @@ func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.Fake } func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { - bucket := &fakeS3Bucket{} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) @@ -512,7 +512,7 @@ func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { name := "backup.not-a-time.sql" bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} c := newMariaDBStrategyTestClient(t, op) - if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + if _, _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { t.Fatal("err = nil, want the unreadable dump time reported") } } @@ -534,9 +534,17 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T if operatorBackupExists(t, r.Client) { t.Error("operator Backup kept, want it deleted") } - if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") { t.Errorf("events = %v, want no ArtifactNotDeleted", events) } + // The listing may have reached other storage than the dump went to, as + // after an endpoint move, so the absence is reported with where it was + // looked for rather than logged as a delete. + if joined := strings.Join(events, "\n"); !hasEvent(events, "Normal ArtifactAlreadyAbsent") || + !strings.Contains(joined, "s3://bkt/"+cleanupPrefix+cleanupDumpName) || !strings.Contains(joined, "through s3.example:9000") { + t.Errorf("events = %v, want a Normal ArtifactAlreadyAbsent naming the key and the endpoint listed", events) + } } // A run started after the recorded dump was uploaded cannot have written it. From d57ebbe808e831637062a920ba9f12fd5580a541 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:09:15 +0300 Subject: [PATCH 11/18] docs(backups): state what the MariaDB dump delete does not cover The comment on who deletes a MariaDB Backup listed namespace teardown without saying that teardown can remove the operator Backup first, in which case the dump is left behind with a Warning, so it promised a delete the code does not make. The remover contract claimed a missing key is never an error, which holds only for a backend that answers the delete with 204, the one status minio-go accepts. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/artifact_size.go | 4 +++- internal/backupcontroller/backup_controller.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 40d827c62..97a67a729 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -39,7 +39,9 @@ type s3ObjectLister interface { } type s3ObjectRemover interface { - // RemoveObject deletes key; a key that does not exist is not an error. + // RemoveObject deletes key. S3 answers 204 for a key that does not + // exist and minio-go takes 204 as success; a backend answering a + // missing key with any other status makes it an error. RemoveObject(ctx context.Context, bucket, key string) error } diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index d8c69bf1c..a632f851d 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -142,7 +142,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // a dump on the platform bucket is deleted here or never. A dump in // a tenant's own bucket is deleted too: the Backup proves it owns // it, and tenants cannot delete Backups, so the delete comes from an - // administrator or from namespace teardown (see cleanupMariaDBBackup). + // administrator or from namespace teardown. Teardown can remove the + // operator Backup first, and the dump is then left behind with a + // Warning, since its storage is read from it (see cleanupMariaDBBackup). return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the From 14a842ad168c0fbe3c650a6ff541c4af163ee114 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:15:18 +0300 Subject: [PATCH 12/18] fix(backups): find a MariaDB operator Restore the cache has not seen The MariaDB restore creates its operator Restore once and then follows it, finding it by label in the informer cache. Nothing watches operator Restores, so a reconcile that ran before the new Restore reached the cache saw none: it created a second one, or, now that a restore refuses a Backup being deleted, failed the RestoreJob while its Restore was still running, after which the cleanup deleted the dump that Restore was reading. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 38 +++++++++++++++++++ .../mariadbstrategy_controller.go | 6 ++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 218c91de6..1919d41c9 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -222,6 +222,44 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { f.wantFailed(t, "src-bk", "is gone") }) } + // The refusal applies only before the operator Restore exists. One + // created by an earlier reconcile but not yet in the cache must be + // found and followed, or the RestoreJob fails under a running restore + // and the cleanup deletes the dump that restore reads. + t.Run("an operator Restore the cache has not seen yet is followed", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Get(ctx, client.ObjectKeyFromObject(live), live); err != nil || live.DeletionTimestamp.IsZero() { + t.Fatalf("Backup not left being deleted: %v", err) + } + live.ResourceVersion = "" + apiServer := newMariaDBStrategyTestClient(t, live, &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }}) + f.r.APIReader = apiServer + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed under a running restore: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want the existing one followed", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index e3e00dbec..b97f7548b 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -756,10 +756,12 @@ func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *ba } // findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled -// with the RestoreJob, if a previous reconcile already created one. +// with the RestoreJob, if a previous reconcile already created one. It is +// read uncached: one the cache has not caught up with would be created +// again, or, once its Backup is being deleted, refused while it runs. func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} - if err := r.List(ctx, list, + if err := r.apiReader().List(ctx, list, client.InNamespace(rj.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: rj.Name, From ee6404a0c88af0850910b9bbb25ad8cc292f30ed Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:28:45 +0300 Subject: [PATCH 13/18] fix(backups): retry a failed lookup of a MariaDB operator Restore The lookup now goes to the API server, which can time out or throttle, and a failed lookup failed the RestoreJob. It says nothing about whether the operator Restore exists, so the RestoreJob could be failed while its Restore was running, and the Backup's cleanup would then stop waiting for it and delete the dump the Restore reads. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 22 +++++++++++++++++++ .../mariadbstrategy_controller.go | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 1919d41c9..5ec96e663 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -260,6 +260,28 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { t.Errorf("created %d operator Restores; want the existing one followed", len(got)) } }) + // A failed lookup says nothing about whether the operator Restore + // exists, so the RestoreJob must not be failed under one that runs. + t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil { + t.Error("err = nil, want the failed lookup returned for a retry") + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores without knowing whether one exists", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index b97f7548b..28a24a14e 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -598,7 +598,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + // Retried: failing the RestoreJob here could fail it under an + // operator Restore that is running. + return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err) } // The dump is chosen once, as the operator Restore is created; one that // already exists has made its choice. From 82468082890a9e6a587ae6c94f7ac02622f6610d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:30:41 +0300 Subject: [PATCH 14/18] fix(backups): hold a MariaDB dump while an operator Restore reads it The cleanup waited for a restore by the phase of its RestoreJob, but the phase is not the restore: a RestoreJob fails on its deadline while the operator Restore it created keeps running, and that Restore still reads the dump. Deleting the dump then made mariadb-operator restore the nearest other one. The cleanup now also holds the Backup while an operator Restore of its operator Backup has not been marked Complete, which mariadb-operator does once the Restore's Job has completed or failed, so what keeps the dump is the restore that reads it rather than the status of the request. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 32 +++++-- .../mariadbstrategy_cleanup_test.go | 83 +++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 25775185b..17d612955 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -59,15 +59,15 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac // The operator Restore reads the dump through the operator Backup, so // neither may go while a restore from this Backup runs. - restore, err := r.activeRestoreOf(ctx, backup) + restore, err := r.activeRestoreOf(ctx, backup, opName) if err != nil { return ctrl.Result{}, err } if restore != "" { - logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restore", restore) if r.Recorder != nil { r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", - "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + "%s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", restore, mariadbSkipArtifactCleanupAnnotation) } return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil @@ -208,9 +208,14 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli return "", true, nil } -// activeRestoreOf returns the name of a RestoreJob that restores from -// backup and has not finished, or "". -func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { +// activeRestoreOf names a restore from backup that has not finished: a +// RestoreJob of it, or an operator Restore reading the dump of opName, the +// operator Backup it was made of; "" when there is none. The operator +// Restore is checked on its own because a RestoreJob can fail, on its +// deadline, while the Restore it created keeps reading the dump. +// mariadb-operator marks a Restore Complete=True once its Job has +// completed or failed. +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup, opName string) (string, error) { list := &backupsv1alpha1.RestoreJobList{} if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) @@ -221,7 +226,20 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv continue } if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { - return rj.Name, nil + return "RestoreJob " + rj.Name, nil + } + } + restores := &mariadbtypes.RestoreList{} + if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) + } + for i := range restores.Items { + restore := &restores.Items[i] + if restore.Spec.BackupRef == nil || restore.Spec.BackupRef.Name != opName { + continue + } + if !apimeta.IsStatusConditionTrue(restore.Status.Conditions, mariadbtypes.ConditionTypeComplete) { + return "k8s.mariadb.com/Restore " + restore.Name, nil } } return "", nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index da9e0902e..ec2dea1a8 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -753,6 +753,89 @@ func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase } } +// operatorRestoreOf is a k8s.mariadb.com/Restore reading the dump of the +// operator Backup backupRef, with the given Complete condition. +func operatorRestoreOf(name, backupRef string, complete *metav1.Condition) *mariadbtypes.Restore { + restore := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}} + if backupRef != "" { + restore.Spec.BackupRef = &mariadbtypes.BackupReference{Name: backupRef} + } + if complete != nil { + restore.Status.Conditions = []metav1.Condition{*complete} + } + return restore +} + +func restoreComplete(status metav1.ConditionStatus, reason string) *metav1.Condition { + return &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: reason, LastTransitionTime: metav1.Now()} +} + +// The RestoreJob's phase is not the restore: one failed on its deadline +// leaves its operator Restore running, and that Restore still reads the +// dump. Until the operator marks it Complete, as it does a finished or a +// failed Job, the dump stays. +func TestCleanupMariaDBBackup_WaitsForAnOperatorRestoreOfItsDump(t *testing.T) { + held := map[string]*metav1.Condition{ + "running": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobRunning), + "suspended": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobSuspended), + "not yet reconciled": nil, + } + for name, cond := range held { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseFailed), operatorRestoreOf("rj-abcde", "op-bk", cond)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the operator Restore runs", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), "k8s.mariadb.com/Restore rj-abcde") { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the operator Restore", events) + } + }) + } + released := map[string]*mariadbtypes.Restore{ + "completed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobComplete)), + "failed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobFailed)), + "of another operator Backup": operatorRestoreOf("rj-abcde", "op-other", nil), + "from storage, not a Backup": operatorRestoreOf("rj-abcde", "", nil), + } + for name, restore := range released { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), restore) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) + } + t.Run("one the cache has not seen yet", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, operatorRestoreOf("rj-abcde", "op-bk", nil)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) +} + func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { t.Run(string(phase), func(t *testing.T) { From 0195a66faeea7a143ccf3aea2ba2f95df67d9178 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:35:33 +0300 Subject: [PATCH 15/18] fix(backups): release a MariaDB Backup when the operator CRDs are gone Listing the operator Restores of a Backup before reading its operator Backup made the cleanup fail on a cluster without the mariadb-operator CRDs, retried for good, so the Backup and its namespace never went away. The operator Backup read that follows already releases the Backup in that case; with no Restore kind no restore can run, so the listing now lets the cleanup reach it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/mariadbstrategy_cleanup.go | 5 +++++ .../backupcontroller/mariadbstrategy_cleanup_test.go | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 17d612955..3ff8adfbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -231,6 +231,11 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv } restores := &mariadbtypes.RestoreList{} if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + // Without the operator's CRDs no Restore can run; the operator + // Backup read that follows releases the Backup. + if apimeta.IsNoMatchError(err) { + return "", nil + } return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) } for i := range restores.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ec2dea1a8..e9a90029b 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -1148,6 +1148,9 @@ func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { } } +// Without the mariadb-operator CRDs every k8s.mariadb.com read fails, the +// listing of operator Restores included: no Restore can run then, so the +// Backup is released rather than held for good. func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { bucket := &fakeS3Bucket{} c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ @@ -1157,6 +1160,12 @@ func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { } return c.Get(ctx, key, obj, opts...) }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Restore").GroupKind()} + } + return c.List(ctx, list, opts...) + }, }) r, rec := cleanupReconcilerWithClient(c, bucket) res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) From f2507b4b84f905e13c0dea1565c64f9cf5699a7e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:35:33 +0300 Subject: [PATCH 16/18] docs(backups): name what the MariaDB cleanup reads uncached The reader's comment named only RestoreJobs, and the restore path said the cleanup holds a Backup only for RestoreJobs it can see, while the cleanup now lists both RestoreJobs and operator Restores straight from the API server. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 5 +++-- internal/backupcontroller/mariadbstrategy_controller.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index a632f851d..b67c640ab 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,8 +44,9 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs of a Backup through it: a restore that started before - // the delete may not be in the cache yet. Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup through it: a + // restore that started before the delete may not be in the cache yet. + // Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 28a24a14e..edff9d187 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -674,7 +674,8 @@ func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj // Deleting a Backup deletes its dump, and the operator restores the // nearest other dump once its own is gone, so a restore must not start // from a Backup being deleted. The copy in hand can predate the delete; - // the cleanup holds the Backup only for RestoreJobs it can already see. + // the cleanup holds the Backup only for restores that exist when it + // lists them. live := &backupsv1alpha1.Backup{} if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { if apierrors.IsNotFound(err) { From a6a704768ce0deb6013a6ab33772d46d0e16da6e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:41:03 +0300 Subject: [PATCH 17/18] test(backups): pin a failed RestoreJob listing in the MariaDB cleanup A RestoreJob listing that fails says nothing about whether a restore reads the dump. Swallowing that error left the whole suite green, while it would let the cleanup delete the dump under a running restore. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index e9a90029b..354bf83f3 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -866,6 +866,21 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) } }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), From b3ca798bc438f0854d624b19d11fb1bab67a73d7 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 12 Nov 2026 17:01:29 +0300 Subject: [PATCH 18/18] fix(backups): read the runs that may own a MariaDB dump uncached Before deleting a dump the cleanup checks every other operator Backup on the same prefix that could have written it, and it read them from the informer cache. A run whose operator Backup had not reached the cache yet but had already uploaded its dump was missed, and its dump deleted as this Backup's. Both the recorded and the legacy paths now list the runs from the API server, as the restore checks already do. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/backup_controller.go | 7 ++--- .../mariadbstrategy_cleanup.go | 4 +-- .../mariadbstrategy_cleanup_test.go | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index b67c640ab..5e6292635 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,9 +44,10 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs and operator Restores of a Backup through it: a - // restore that started before the delete may not be in the cache yet. - // Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup, and the operator + // Backups that may own its dump, through it: a restore that started + // before the delete, or a run that has already uploaded its dump, may + // not be in the cache yet. Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 3ff8adfbe..912bb0038 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -120,7 +120,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac } absent := false if name != "" { - other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name) absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) @@ -130,7 +130,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil } } else { - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 354bf83f3..1d2f10cbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -547,6 +547,33 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T } } +// The runs that may own a dump are read from the API server: a run whose +// operator Backup has not reached the cache yet can already have uploaded +// its dump, and missing it would delete that dump as this Backup's. +func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + unseen := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-unseen" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""} + for name, recorded := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + backup := cleanupBackup(recorded) + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + // A run started after the recorded dump was uploaded cannot have written it. func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { key := cleanupPrefix + cleanupDumpName