diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 6376c8e87..88ba71e2f 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -39,6 +39,13 @@ type s3ObjectLister interface { StatObject(ctx context.Context, bucket, key string) (int64, error) } +type s3ObjectRemover interface { + // RemoveObject deletes key. S3 answers 204 for a key that does not + // exist and minio-go takes 204 as success; a backend answering a + // missing key with any other status makes it an error. + RemoveObject(ctx context.Context, bucket, key string) error +} + // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { @@ -67,8 +74,10 @@ type objectStore struct { // reader reads Secrets, which the manager's client fetches uncached, // and lists operator Backups, which it serves from the cache. reader client.Reader - // newLister overrides the S3 client; nil means a real minio client. - newLister func(conn s3Connection) (s3ObjectLister, error) + // newLister and newRemover override the S3 client; nil means a real + // minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) + newRemover func(conn s3Connection) (s3ObjectRemover, error) } func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { @@ -78,6 +87,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { return newMinioObjectLister(conn) } +func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) { + if s.newRemover != nil { + return s.newRemover(conn) + } + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { @@ -94,6 +114,14 @@ func (r *BackupJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *RestoreJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + +func (r *BackupReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } @@ -103,6 +131,14 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + +func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) { opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -168,6 +204,10 @@ func (l *minioObjectLister) StatObject(ctx context.Context, bucket, key string) return info.Size, nil } +func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error { + return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}) +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 6741fbaf6..bc49baa6b 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -39,9 +39,22 @@ type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds - // into the Backup's namespace before spawning its best-effort delete Job. + // CredentialsConfig locates the platform backup credentials: the Job-based + // cleanups project them into the Backup's namespace, and the MariaDB + // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + // APIReader is the manager's uncached reader. The MariaDB cleanup lists + // the RestoreJobs and operator Restores of a Backup, and the operator + // Backups that may own its dump, through it: a restore that started + // before the delete, or a run that has already uploaded its dump, may + // not be in the cache yet. Wired in SetupWithManager. + APIReader client.Reader + + // newS3ObjectLister and newS3ObjectRemover override the S3 client the + // MariaDB cleanup deletes a dump with; nil means a real minio client. + // Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) + newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error) } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -126,22 +139,20 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: - // Cozystack Backup deletion does NOT delete the operator-side - // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. - // Lifecycle of that CR is owned by mariadb-operator's - // MaxRetention setting and by tenants / explicit teardown flows; - // the controller's RBAC has no delete verb on - // k8s.mariadb.com/backups for that reason. Surfacing this as an - // explicit branch (rather than falling through to the Velero - // default) keeps the contract honest: the MariaDB driver does - // not own the archive, so it does not delete it on CR removal. - // Mirrors the Altinity branch's "we do not own S3" stance. - return ctrl.Result{}, nil + // mariadb-operator's maxRetention prunes only from inside the next + // backup run of the same prefix and a Plan never deletes a Backup, so + // a dump on the platform bucket is deleted here or never. A dump in + // a tenant's own bucket is deleted too: the Backup proves it owns + // it, and tenants cannot delete Backups, so the delete comes from an + // administrator or from namespace teardown. Teardown can remove the + // operator Backup first, and the dump is then left behind with a + // Warning, since its storage is read from it (see cleanupMariaDBBackup). + return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the - // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup + // archive" contract as Altinity / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer @@ -154,7 +165,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own - // the archive" contract as Altinity / MariaDB — the explicit + // the archive" contract as Altinity — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. @@ -294,7 +305,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.APIReader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) } + +// apiReader returns the uncached reader. Reconcilers built by hand in unit +// tests have none and fall back to their fake Client, which has no cache to +// bypass. +func (r *BackupReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} diff --git a/internal/backupcontroller/backup_controller_test.go b/internal/backupcontroller/backup_controller_test.go index 579cb5fe6..d77f5d208 100644 --- a/internal/backupcontroller/backup_controller_test.go +++ b/internal/backupcontroller/backup_controller_test.go @@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) { // TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same // invariant as the Altinity test above for the MariaDB strategy. The -// dispatcher must explicitly route MariaDB-strategy Backups to a no-op -// cleanup rather than fall through to the Velero default - the MariaDB -// driver does not own the operator-side k8s.mariadb.com/Backup CR or the -// S3/PVC archive behind it (rbac.yaml grants no delete verb on -// k8s.mariadb.com/backups for that reason). The test seeds the Velero +// dispatcher must route MariaDB-strategy Backups to their own cleanup +// rather than fall through to the Velero default. The test seeds the Velero // metadata keys plus a matching velero.io/Backup; with the explicit branch // no DeleteBackupRequest is synthesised, and the Velero Backup survives. // Without the branch the default falls through to cleanupVeleroBackup, diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 6397325ca..4e0525517 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -39,19 +39,19 @@ type mariadbDump struct { LastModified time.Time } -// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for -// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump -// cannot be found or told apart: an unknown size must not be reported as a -// measured one. -func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 { +// attributedMariaDBDump returns the dump mariadb-operator uploaded for +// mdbBackup. ok is false when the dump cannot be found or told apart, and +// the Backup then reports neither a size nor a dump: an unknown dump must +// not be reported as a measured one. +func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (dump mariadbDump, ok bool) { ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt) if err != nil { - getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) - return 0 + getLogger(ctx).Info("MariaDB dump not attributed; leaving its size and name unset", "backup", mdbBackup.Name, "error", err) + return mariadbDump{}, false } - return dump.Size + return dump, true } // findMariaDBDump lists the bucket mdbBackup wrote to and returns the one @@ -60,22 +60,26 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbB // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + lister, _, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return mariadbDump{}, err + } + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt, false) +} + +// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, +// for a caller that reaches the bucket by another endpoint than the one +// mdbBackup names. withUnfinished lets runs that have not completed contest +// the dump too, as neighbourMariaDBRuns describes. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time, withUnfinished bool) (mariadbDump, error) { s3 := mdbBackup.Spec.Storage.S3 if s3 == nil { return mariadbDump{}, errors.New("the operator Backup does not store to S3") } - others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + others, err := neighbourMariaDBRuns(ctx, reader, mdbBackup, withUnfinished) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } - conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) - if err != nil { - return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) - } - lister, err := store.lister(conn) - if err != nil { - return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) - } // Non-recursive, as mariadb-operator lists it: the default prefix is also // the default CNPG destinationPath, so an app of each kind with the same // name shares it and a recursive listing would walk the WAL archive. @@ -88,6 +92,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt return matchMariaDBDump(objects, prefix, own, others) } +// mariadbDumpExists reports whether the dump file name is still in the +// storage mdbBackup wrote to. +func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) { + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return false, err + } + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + for _, obj := range objects { + if obj.Key == key { + return true, nil + } + } + return false, nil +} + +// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials +// its own spec names. +func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return nil, nil, errors.New("the operator Backup does not store to S3") + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) + if err != nil { + return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return nil, nil, fmt.Errorf("build S3 client: %w", err) + } + return lister, s3, nil +} + // mariadbRun is the span in which an operator Backup can have written its // dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. type mariadbRun struct { @@ -95,6 +137,10 @@ type mariadbRun struct { lower, upper time.Time } +// mariadbUnfinishedRunEnd closes the window of a run that has not completed, +// which can still write its dump at any time. +var mariadbUnfinishedRunEnd = time.Date(9999, time.December, 31, 0, 0, 0, 0, time.UTC) + func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { return mariadbRun{ name: b.Namespace + "/" + b.Name, @@ -120,8 +166,8 @@ func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { // dump in the window on one clock only may be this run's own, misplaced by // a skewed clock, which leaves any other candidate unproven. A neighbour's // dump is still taken if this run's own is missing from the listing, or -// misplaced on both clocks, and the neighbour's operator Backup is gone or -// has not completed. +// misplaced on both clocks, and the neighbour's operator Backup is gone or, +// unless others includes unfinished runs, has not completed. func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { var near []mariadbDump for _, obj := range objects { @@ -153,14 +199,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, oth return d, nil } -// finishedMariaDBRuns returns the runs of the other operator Backups that +// neighbourMariaDBRuns returns the runs of the other operator Backups that // write under the same bucket and prefix as b and have completed, whether -// their Job succeeded or failed. One still running is left out: its window -// has no end yet, and an operator Backup stuck short of Complete, as one -// whose Job pod never starts is, would otherwise contest every later run of -// the application for good. The endpoint is not compared, since one has -// several spellings, and a false neighbour only leaves a size unset. -func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { +// their Job succeeded or failed. One still running is left out unless +// withUnfinished is set: its window has no end yet, and an operator Backup +// stuck short of Complete, as one whose Job pod never starts is, would +// otherwise contest every later run of the application for good. That is +// the price of deleting a dump, since a run can upload its dump and prune +// older ones through maxRetention before it completes. The endpoint is not +// compared, since one has several spellings, and a false neighbour only +// keeps a dump unattributed. +func neighbourMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup, withUnfinished bool) ([]mariadbRun, error) { list := &mariadbtypes.BackupList{} if err := reader.List(ctx, list); err != nil { return nil, err @@ -177,6 +226,9 @@ func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbty } cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) if cond == nil || cond.Status != metav1.ConditionTrue { + if withUnfinished { + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, mariadbUnfinishedRunEnd)) + } continue } runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index ed4a2b7bc..239ec296c 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -29,6 +29,14 @@ func dumpKey(prefix string, ts time.Time, compression string) string { return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) } +// probedSize is the size a Backup reports for a probe result. +func probedSize(dump mariadbDump, attributed bool) int64 { + if !attributed { + return 0 + } + return dump.Size +} + // mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a // Backup whose Job succeeded. A failed Job also gets Status=True, with // Reason=JobFailed, so a fixture without the reason is not a success. @@ -105,7 +113,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} - got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed) + got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)) if got != 4242 { t.Errorf("size = %d, want 4242", got) } @@ -130,7 +138,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed)); got != 3 { t.Errorf("size = %d, want 3", got) } if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { @@ -148,7 +156,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed)); got != 4 { t.Errorf("size = %d, want 4", got) } if l.conn.SessionToken != "TOKEN" { @@ -160,7 +168,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 9 { t.Errorf("size = %d, want 9", got) } }) @@ -173,7 +181,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) } }) @@ -182,7 +190,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 5 { t.Errorf("size = %d, want 5", got) } }) @@ -191,7 +199,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } withShortProbeTimeout(t, func() { - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -202,7 +210,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) } }) @@ -211,14 +219,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) t.Run("list error leaves size unset", func(t *testing.T) { l := &fakeS3Lister{err: errors.New("AccessDenied")} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -226,7 +234,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -481,7 +489,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { s3.Prefix = "tenant/db/" s3.Region = "us-east-1" - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute))); got != 31337 { t.Fatalf("size = %d, want 31337", got) } if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { @@ -523,6 +531,69 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { } } +// TestCreateMariaDBBackupArtifact_RecordsDump pins that the Backup names +// its dump only when the probe attributed one to the operator Backup. +func TestCreateMariaDBBackupArtifact_RecordsDump(t *testing.T) { + apps := mariadbapp.GroupName + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := metav1.NewTime(created.Add(3 * time.Minute)) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + own := s3ObjectInfo{Key: "src/backup.2026-10-21T10:01:00Z.gzip.sql", Size: 2048, LastModified: created.Add(2 * time.Minute)} + second := s3ObjectInfo{Key: "src/backup.2026-10-21T10:02:00Z.gzip.sql", Size: 1, LastModified: created.Add(2 * time.Minute)} + offset := s3ObjectInfo{Key: "src/backup.2026-10-21T14:01:00+04:00.sql", Size: 7, LastModified: created.Add(2 * time.Minute)} + + cases := []struct { + name string + lister *fakeS3Lister + wantFile, wantTime string + }{ + {name: "an attributed dump is recorded", lister: &fakeS3Lister{objects: []s3ObjectInfo{own}}, + wantFile: "backup.2026-10-21T10:01:00Z.gzip.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "a dump named with an offset is timed in UTC", lister: &fakeS3Lister{objects: []s3ObjectInfo{offset}}, + wantFile: "backup.2026-10-21T14:01:00+04:00.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "two dumps in the lifetime record none", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second}}}, + {name: "no dump in the lifetime records none", lister: &fakeS3Lister{}}, + {name: "a failed listing records none", lister: &fakeS3Lister{err: errors.New("AccessDenied")}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, created) + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-dump"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + r := reconcilerWithLister(t, tc.lister, job, s3CredsSecret("tenant", "creds")) + if _, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered); err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + persisted := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-dump"}, persisted); err != nil { + t.Fatalf("get Backup: %v", err) + } + md := persisted.Spec.DriverMetadata + file, hasFile := md[mariadbDumpFileKey] + at, hasTime := md[mariadbDumpTimeKey] + if tc.wantFile == "" { + if hasFile || hasTime { + t.Errorf("driverMetadata = %v; want no dump recorded", md) + } + return + } + if file != tc.wantFile || at != tc.wantTime { + t.Errorf("recorded dump %q at %q; want %q at %q", file, at, tc.wantFile, tc.wantTime) + } + }) + } +} + // TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the // probe reads where the operator Backup wrote its dump, not where the // strategy, edited while the backup ran, now points. diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go new file mode 100644 index 000000000..5ec96e663 --- /dev/null +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// The source run of the restore fixture: operator Backup op-bk of +// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing +// under tenant/src/ with mdbRestoreDump as its dump. +var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + +const ( + mdbRestorePrefix = "tenant/src/" + mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql" + mdbRestoreTime = "2026-10-21T10:00:05Z" + mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql" +) + +type mariadbRestoreFixture struct { + c client.Client + r *RestoreJobReconciler + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup + src *mariadbtypes.Backup + lister *fakeS3Lister + events *record.FakeRecorder +} + +// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose +// driverMetadata is driverMD plus the operator Backup's name, into the +// existing application target. extra objects are seeded too. +func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture { + t.Helper() + apps := mariadbapp.GroupName + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/src" + src := operatorBackupStoringTo(t, s3, mdbRestoreT0) + src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))} + + md := map[string]string{mariadbBackupNameKey: src.Name} + for k, v := range driverMD { + md[k] = v + } + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)), + DriverMetadata: md, + }, + } + started := metav1.Now() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: backup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}} + objs := append([]client.Object{src, backup, rj, target}, extra...) + c := newMariaDBStrategyTestClient(t, objs...) + fetched := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + events := record.NewFakeRecorder(10) + return &mariadbRestoreFixture{ + c: c, + rj: fetched, + backup: backup, + src: src, + lister: lister, + events: events, + r: &RestoreJobReconciler{ + Client: c, + Recorder: events, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + lister.conn = conn + return lister, nil + }, + }, + } +} + +func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result { + t.Helper() + res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + return res +} + +func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore { + t.Helper() + list := &mariadbtypes.RestoreList{} + if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { + t.Fatalf("list operator Restores: %v", err) + } + return list.Items +} + +// wantFailed asserts the RestoreJob failed with a message holding every +// fragment, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) { + t.Helper() + if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase) + } + for _, s := range fragments { + if !strings.Contains(f.rj.Status.Message, s) { + t.Errorf("message %q does not contain %q", f.rj.Status.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantWaiting asserts the RestoreJob is kept waiting on the dump check +// with Ready=False saying why, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) { + t.Helper() + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message) + } + if res.RequeueAfter != mariadbDumpCheckRetryInterval { + t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval) + } + persisted := &backupsv1alpha1.RestoreJob{} + if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" { + t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond) + } + for _, s := range fragments { + if !strings.Contains(cond.Message, s) { + t.Errorf("Ready message %q does not contain %q", cond.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantRestore asserts exactly one operator Restore of op-bk into target, +// asking for targetTime. +func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) { + t.Helper() + got := f.restores(t) + if len(got) != 1 { + t.Fatalf("created %d operator Restores; want one", len(got)) + } + spec := got[0].Spec + if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") { + t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target")) + } + if spec.TargetRecoveryTime != targetTime { + t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime) + } +} + +func recordedDump(name string) map[string]string { + return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} +} + +// A Backup being deleted is deleting its dump, so a restore that starts +// now can find it gone once the operator reads it, and the operator then +// restores the nearest other dump. The Backup is read from the API server, +// since the copy the reconcile was handed can predate the delete. +func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + present := func() *fakeS3Lister { + return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}} + } + cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil} + for name, md := range cases { + t.Run(name+", being deleted", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "being deleted") + }) + t.Run(name+", already gone", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "is gone") + }) + } + // The refusal applies only before the operator Restore exists. One + // created by an earlier reconcile but not yet in the cache must be + // found and followed, or the RestoreJob fails under a running restore + // and the cleanup deletes the dump that restore reads. + t.Run("an operator Restore the cache has not seen yet is followed", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Get(ctx, client.ObjectKeyFromObject(live), live); err != nil || live.DeletionTimestamp.IsZero() { + t.Fatalf("Backup not left being deleted: %v", err) + } + live.ResourceVersion = "" + apiServer := newMariaDBStrategyTestClient(t, live, &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }}) + f.r.APIReader = apiServer + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed under a running restore: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want the existing one followed", len(got)) + } + }) + // A failed lookup says nothing about whether the operator Restore + // exists, so the RestoreJob must not be failed under one that runs. + t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil { + t.Error("err = nil, want the failed lookup returned for a retry") + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores without knowing whether one exists", len(got)) + } + }) + t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable") + }) +} + +func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + + t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}, + {Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" { + t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump) + } + }) + + t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix) + }) + + t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.") + }) + + t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials") + if l.calls != 0 { + t.Errorf("S3 listed %d times without credentials", l.calls) + } + }) + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var res ctrl.Result + var err error + withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error()) + }) + + t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute)) + f.rj.Status.StartedAt = &long + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after") + }) + + t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds) + f.reconcile(t) + f.wantFailed(t, `"dump.sql"`) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a name that is not a dump", l.calls) + } + }) + + t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing) + res := f.reconcile(t) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls) + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval { + t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter) + } + if got := f.restores(t); len(got) != 1 { + t.Errorf("%d operator Restores; want only the existing one", len(got)) + } + }) +} + +// TestReconcileMariaDBRestore_LegacyBackup drives Backups that record no +// dump: one proven to belong to the source run is restored by its +// timestamp, and anything else restores the newest dump with a Warning. +func TestReconcileMariaDBRestore_LegacyBackup(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + own := s3ObjectInfo{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)} + newer := s3ObjectInfo{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)} + second := s3ObjectInfo{Key: mdbRestorePrefix + "backup.2026-10-21T10:00:30Z.gzip.sql", LastModified: mdbRestoreT0.Add(35 * time.Second)} + + t.Run("a dump proven to be the run's own is restored by its timestamp", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: []s3ObjectInfo{own, newer}}, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if ev := drainEvents(f.events); len(ev) != 0 { + t.Errorf("events %q; want none for a proven dump", ev) + } + }) + + cases := []struct { + name string + lister *fakeS3Lister + cause string + }{ + {name: "two dumps in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second, newer}}, cause: "cannot be told apart"}, + {name: "no dump in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{newer}}, cause: "no dump in the bucket belongs"}, + {name: "a storage that cannot be read", lister: &fakeS3Lister{err: errors.New("Access Denied.")}, cause: "Access Denied."}, + } + wantBestEffort := func(t *testing.T, f *mariadbRestoreFixture, cause string) { + t.Helper() + f.wantRestore(t, "") + ev := drainEvents(f.events) + if len(ev) != 1 || !strings.HasPrefix(ev[0], "Warning BestEffortDump ") || !strings.Contains(ev[0], cause) || !strings.Contains(ev[0], "src-bk records no dump") { + t.Errorf("events %q; want one BestEffortDump warning naming the Backup and %q", ev, cause) + } + } + for _, tc := range cases { + t.Run(tc.name+" restores the newest dump with a warning", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, tc.lister, creds) + f.reconcile(t) + wantBestEffort(t, f, tc.cause) + }) + } + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var err error + withShortDumpCheckTimeout(t, func() { _, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + wantBestEffort(t, f, context.DeadlineExceeded.Error()) + }) +} + +// TestReconcileMariaDBRestore_FailedBackupRun drives a Backup made of an +// operator Backup whose Job failed: there is no dump of its own to +// restore, whatever the Backup records. +func TestReconcileMariaDBRestore_FailedBackupRun(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + failed := func(status metav1.ConditionStatus) []metav1.Condition { + return []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: mariadbtypes.ConditionReasonJobFailed, + Message: "Failed", LastTransitionTime: metav1.NewTime(mdbRestoreT0.Add(3 * time.Minute)), + }} + } + listing := []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}} + + cases := []struct { + name string + driverMD map[string]string + status metav1.ConditionStatus + }{ + {name: "Complete=True/JobFailed with no recorded dump", status: metav1.ConditionTrue}, + {name: "Complete=True/JobFailed with a recorded dump", driverMD: recordedDump(mdbRestoreDump), status: metav1.ConditionTrue}, + {name: "Complete=False/JobFailed", status: metav1.ConditionFalse}, + } + for _, tc := range cases { + t.Run(tc.name+" fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: listing} + f := newMariaDBRestoreFixture(t, tc.driverMD, l, creds) + f.src.Status.Conditions = failed(tc.status) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + f.wantFailed(t, "k8s.mariadb.com/Backup tenant/op-bk failed", "left no dump to restore") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a failed backup run", l.calls) + } + }) + } + + t.Run("an operator Restore created earlier is followed", func(t *testing.T) { + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: listing}, creds, existing) + f.src.Status.Conditions = failed(metav1.ConditionTrue) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("RestoreJob failed although its operator Restore was already created: %s", f.rj.Status.Message) + } + }) +} + +func drainEvents(r *record.FakeRecorder) []string { + var out []string + for { + select { + case e := <-r.Events: + out = append(out, e) + default: + return out + } + } +} + +// withShortDumpCheckTimeout shortens the restore's dump check timeout and +// fails the test if fn is not back well before the default would expire. +func withShortDumpCheckTimeout(t *testing.T, fn func()) { + t.Helper() + saved := mariadbRestoreDumpCheckTimeout + mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond + defer func() { mariadbRestoreDumpCheckTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the dump check did not give up at its timeout") + } +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go new file mode 100644 index 000000000..912bb0038 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup" + + // Retry cadence while a dump cannot be deleted or a restore from the + // Backup is still running. + mariadbCleanupRetryInterval = 30 * time.Second +) + +// mariadbDumpCleanupTimeout bounds the object-store round trips of one +// cleanup attempt. The Backup controller has a single worker, so an +// unreachable endpoint stalls every other Backup for this long. A variable +// so tests can shorten it. +var mariadbDumpCleanupTimeout = 10 * time.Second + +// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then +// the operator Backup that wrote it. Every run of an application writes +// under one prefix, so the only key deleted is one proven to be this +// Backup's: the dump it records, unless contestedMariaDBDump finds another +// run that could have written it, or, for a Backup taken before dumps were +// recorded, the one findMariaDBDump attributes to its operator Backup. When +// no dump is proven the Backup is released with a Warning and nothing is +// deleted, the operator Backup included, since it is what any later +// attribution would start from. A failed delete holds the Backup and +// retries, as the Redis and Rabbitmq cleanups do. +func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + logger := getLogger(ctx) + if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" { + logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation) + return ctrl.Result{}, nil + } + + opName := backup.Spec.DriverMetadata[mariadbBackupNameKey] + if opName == "" { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil + } + + // The operator Restore reads the dump through the operator Backup, so + // neither may go while a restore from this Backup runs. + restore, err := r.activeRestoreOf(ctx, backup, opName) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restore", restore) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", + "%s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + restore, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil + } + + op := &mariadbtypes.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", + fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil + } + return ctrl.Result{}, err + } + + s3 := op.Spec.Storage.S3 + if s3 == nil { + return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName), + "PVC and Volume storage is not reachable from the controller"), nil + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix) + + // A failed Job is Complete=True too, with reason JobFailed, and wrote no + // dump: whatever its window holds is another run's. + if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil || + cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil + } + + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name != "" { + if _, ok := parseMariaDBLogicalBackupFile(name); !ok { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil + } + } + + conn, err := r.mariadbCleanupConnection(ctx, op) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err)) + } + + s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) + defer cancel() + store := r.objectStore() + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + absent := false + if name != "" { + other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name) + absent = !found + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + if other != "" { + return r.releaseMariaDBCleanup(ctx, backup, location+name, + fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil + } + } else { + dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true) + if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil + } + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, err) + } + name = dump.Name + // Recorded first: once the dump is gone a retry could not match it + // again, and could match a neighbour's instead. + base := backup.DeepCopy() + backup.Spec.DriverMetadata[mariadbDumpFileKey] = name + backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err) + } + } + + remover, err := store.remover(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + policy := metav1.DeletePropagationBackground + if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) + } + if absent { + logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent", + "%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted", + location+name, conn.Endpoint, opName) + } + return ctrl.Result{}, nil + } + logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) + return ctrl.Result{}, nil +} + +// contestedMariaDBDump returns the name of another operator Backup on op's +// bucket and prefix whose run, finished or not, could have written the dump +// name, or "" when none could, and whether the listing found the dump. The +// size probe that recorded name let only runs finished by then contest it, +// and a run can upload its dump and prune op's through maxRetention before +// it completes. A dump no longer in the bucket is contested by nobody: +// deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) { + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + } + s3 := op.Spec.Storage.S3 + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) + if i < 0 { + return "", false, nil + } + others, err := neighbourMariaDBRuns(ctx, reader, op, true) + if err != nil { + return "", true, fmt.Errorf("list operator Backups: %w", err) + } + d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} + for _, other := range others { + if other.placesOnEitherClock(d) { + return other.name, true, nil + } + } + return "", true, nil +} + +// activeRestoreOf names a restore from backup that has not finished: a +// RestoreJob of it, or an operator Restore reading the dump of opName, the +// operator Backup it was made of; "" when there is none. The operator +// Restore is checked on its own because a RestoreJob can fail, on its +// deadline, while the Restore it created keeps reading the dump. +// mariadb-operator marks a Restore Complete=True once its Job has +// completed or failed. +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup, opName string) (string, error) { + list := &backupsv1alpha1.RestoreJobList{} + if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list RestoreJobs: %w", err) + } + for i := range list.Items { + rj := &list.Items[i] + if rj.Spec.BackupRef.Name != backup.Name { + continue + } + if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + return "RestoreJob " + rj.Name, nil + } + } + restores := &mariadbtypes.RestoreList{} + if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + // Without the operator's CRDs no Restore can run; the operator + // Backup read that follows releases the Backup. + if apimeta.IsNoMatchError(err) { + return "", nil + } + return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) + } + for i := range restores.Items { + restore := &restores.Items[i] + if restore.Spec.BackupRef == nil || restore.Spec.BackupRef.Name != opName { + continue + } + if !apimeta.IsStatusConditionTrue(restore.Status.Conditions, mariadbtypes.ConditionTypeComplete) { + return "k8s.mariadb.com/Restore " + restore.Name, nil + } + } + return "", nil +} + +// mariadbCleanupConnection returns how to reach the bucket op wrote to. A +// dump on the platform bucket, which op names by its bucket and by the +// Secret the platform credentials are projected to, is reached as the +// platform reaches it now, so a Backup taken before the endpoint moved is +// still deleted. Any other bucket, or the platform bucket while the +// platform credentials are missing or malformed, is reached as op recorded +// it; a delete that fails there is handled by retryMariaDBCleanup like any +// other. A failed read of the platform credentials is retried instead: the +// recorded endpoint may be the one the platform moved off, where the dump +// would be found missing and reported deleted. +func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { + if r.CredentialsConfig.IsEnabled() { + conn, bucket, err := r.platformS3Connection(ctx) + var projErr *ProjectionError + switch { + case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError: + return s3Connection{}, err + case err != nil: + getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) + case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: + // A strategy can pin a private CA for the platform bucket; the + // platform credentials carry none. + if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil { + ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key) + if err != nil { + return s3Connection{}, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil + } + } + return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3) +} + +// platformS3Connection reads the platform's backup bucket from the Secret +// cozy-backups-creds is projected from, and its TLS from the scheme of +// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets +// tls.enabled. It reads the source rather than projecting it: projecting +// writes a Secret, which a terminating namespace refuses. +func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) { + cfg := r.CredentialsConfig + src := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing, + Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)} + } + return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)} + } + creds, err := parseSourceSecret(src) + if err != nil { + return s3Connection{}, "", err + } + if creds.endpoint == "" { + creds.endpoint = stripScheme(cfg.Endpoint) + } + if creds.endpoint == "" { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)} + } + if creds.region == "" { + creds.region = cfg.Region + } + return s3Connection{ + Endpoint: creds.endpoint, + Secure: strings.HasPrefix(cfg.Endpoint, "https://"), + Region: creds.region, + AccessKey: creds.accessKey, + SecretKey: creds.secretKey, + }, creds.bucket, nil +} + +// retryMariaDBCleanup holds the Backup after a failed attempt and says so, +// unless its namespace is going away, where nothing it needs can be read +// any longer and holding it would hold the namespace. +func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) { + terminating, err := r.namespaceTerminating(ctx, backup.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if terminating { + return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil + } + getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed", + "cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump", + location, cause, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil +} + +// releaseMariaDBCleanup lets the Backup go without deleting anything further +// and records a Warning naming what was left, so the give-up is visible. +func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result { + getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason) + } + return ctrl.Result{} +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go new file mode 100644 index 000000000..7efb3046f --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -0,0 +1,1379 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// fakeS3Bucket stands in for the object store on both the listing and the +// delete path, and records what was asked of it. +type fakeS3Bucket struct { + objects []s3ObjectInfo + listErr error + removeErr error + + conns []s3Connection + removed []string +} + +func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + return f.objects, f.listErr +} + +func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error { + if f.removeErr != nil { + return f.removeErr + } + f.removed = append(f.removed, bucket+"/"+key) + return nil +} + +var ( + cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + cleanupCompleted = cleanupCreated.Add(3 * time.Minute) + cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql" +) + +const cleanupPrefix = "tenant/db/" + +// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of: +// it stored to bkt under tenant/db with the tenant's creds Secret, and its +// Job succeeded. +func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup { + t.Helper() + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + op := operatorBackupStoringTo(t, s3, cleanupCreated) + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))} + return op +} + +// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup +// op-bk, recording dumpFile when it is not empty. +func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup { + apiGroup := strategyv1alpha1.GroupVersion.Group + md := map[string]string{ + mariadbBackupNameKey: "op-bk", + mariadbBackupNamespaceKey: "tenant", + } + if dumpFile != "" { + md[mariadbDumpFileKey] = dumpFile + } + now := metav1.Now() + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "bk", + DeletionTimestamp: &now, + Finalizers: []string{backupFinalizer}, + }, + Spec: backupsv1alpha1.BackupSpec{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb", + }, + TakenAt: metav1.NewTime(cleanupCompleted), + DriverMetadata: md, + }, + } +} + +func tenantNamespace() *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}} +} + +func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) { + t.Helper() + return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket) +} + +func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: rec, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + }, rec +} + +func operatorBackupExists(t *testing.T, c client.Client) bool { + t.Helper() + err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("get operator Backup: %v", err) + } + return err == nil +} + +func requeues(res ctrl.Result) bool { + return res.Requeue || res.RequeueAfter > 0 +} + +func hasEvent(events []string, prefix string) bool { + for _, e := range events { + if strings.HasPrefix(e, prefix) { + return true + } + } + return false +} + +// assertReleasedUntouched checks a Backup that is let go with nothing +// deleted: no S3 delete, the operator Backup kept, and a Warning saying so. +func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) { + t.Helper() + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 { + t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns) + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } +} + +func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil { + t.Fatalf("err = %v", err) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events) + } +} + +func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil { + t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held") + } +} + +// A legacy Backup records the dump it proved before deleting it, so a retry +// after a failed operator Backup delete finds the key by name instead of +// matching again against a bucket that no longer holds it. +func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + failDelete := true + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err == nil { + t.Fatal("first pass: err = nil, want the failed operator Backup delete returned") + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("Backup gone after a failed operator Backup delete: %v", err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName { + t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata) + } + + // The dump is gone now, as the first pass deleted it. + bucket.objects = nil + failDelete = false + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("second pass: %v", err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted on the retry") + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no claim that the deleted dump was left behind", events) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil { + t.Fatal("err = nil, want the failed record returned so the Backup is held") + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) { + for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + {Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}, + }} + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) { + t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata) + } +} + +// The window is the operator Backup's lifetime, closed by the Backup's +// takenAt: a dump stamped after takenAt plus the skew is not this run's. +func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) { + late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late}, + }} + op := cleanupOperatorBackup(t) + // The operator's own Complete transition is not what bounds the window. + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + cases := map[string][]s3ObjectInfo{ + "no dump in its lifetime": { + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + }, + "two dumps in its lifetime": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}, + {Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)}, + }, + "a dump placed by one clock only": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)}, + }, + } + for name, objects := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: objects} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}} + neighbour := cleanupOperatorBackup(t) + neighbour.Name = "op-neighbour" + neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second)) + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// A run started before this one completed can upload its dump and prune +// this run's through maxRetention before its own operator Backup completes. +// Its dump is then the only one in this run's window, and only its open +// window says it may be its own. +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpAnUnfinishedRunCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(2 * time.Minute) + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, started.Add(90*time.Second), "gzip"), LastModified: started.Add(100 * time.Second)}, + }} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(started) + running.Status.Conditions = conditions + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// An unfinished run contests only what it could have written: a dump +// that predates it, on both clocks, is still this run's. +func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + backup := cleanupBackup("") + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// The size probe that records a dump lets only finished runs contest it, so +// a run still unfinished then, which uploaded its dump and pruned this +// run's through maxRetention, can have its dump recorded here. Deletion +// contests the recorded dump again, with every run as it stands now. +func TestCleanupMariaDBBackup_RecordedDumpARunUnfinishedAtTheProbeCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + "completed since": {mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted.Add(5 * time.Minute)))}, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(started) + other.Status.Conditions = conditions + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// Either clock placing the recorded dump in another run's lifetime is +// enough: the backup pod's clock can be skewed while the store's is not. +func TestCleanupMariaDBBackup_RecordedDumpAnotherRunPlacesOnOneClockIsReleased(t *testing.T) { + stamped := cleanupCreated.Add(20 * time.Second) + uploaded := cleanupCreated.Add(2 * time.Minute) + theirs := dumpKey(cleanupPrefix, stamped, "gzip") + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: uploaded}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(uploaded.Add(30 * time.Second)) + other.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// The listing is by prefix, so a longer key sharing it is not the dump and +// contests nothing. +func TestCleanupMariaDBBackup_RecordedDumpIgnoresAKeyItIsOnlyAPrefixOf(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key + ".tmp", LastModified: cleanupCompleted.Add(2 * time.Hour)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// A name the dump time cannot be read from leaves one clock out of the +// contest, so it is an error rather than a contest on half the evidence. +func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { + op := cleanupOperatorBackup(t) + name := "backup.not-a-time.sql" + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} + c := newMariaDBStrategyTestClient(t, op) + if _, _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + t.Fatal("err = nil, want the unreadable dump time reported") + } +} + +// A retry after the dump was deleted but the operator Backup was not finds +// nothing to contest, whichever run's lifetime the dump's name falls in. +func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T) { + bucket := &fakeS3Bucket{} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCreated) + running.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no ArtifactNotDeleted", events) + } + // The listing may have reached other storage than the dump went to, as + // after an endpoint move, so the absence is reported with where it was + // looked for rather than logged as a delete. + if joined := strings.Join(events, "\n"); !hasEvent(events, "Normal ArtifactAlreadyAbsent") || + !strings.Contains(joined, "s3://bkt/"+cleanupPrefix+cleanupDumpName) || !strings.Contains(joined, "through s3.example:9000") { + t.Errorf("events = %v, want a Normal ArtifactAlreadyAbsent naming the key and the endpoint listed", events) + } +} + +// The runs that may own a dump are read from the API server: a run whose +// operator Backup has not reached the cache yet can already have uploaded +// its dump, and missing it would delete that dump as this Backup's. +func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + unseen := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-unseen" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""} + for name, recorded := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + backup := cleanupBackup(recorded) + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// A run started after the recorded dump was uploaded cannot have written it. +func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } +} + +func TestCleanupMariaDBBackup_RecordedDumpListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// Without the list of runs nothing is known about who else could have +// written the dump, so it is kept and the delete retried. +func TestCleanupMariaDBBackup_RecordedDumpWithUnreadableRunsRetries(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.BackupList); ok { + return errors.New("apiserver unavailable") + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// The whole sequence: run B uploads its dump and prunes this run's before +// B completes, this run's Backup is created and probed, then deleted. B's +// dump must survive whatever the probe recorded. +func TestMariaDBBackup_ProbeThenCleanupKeepsADumpAnUnfinishedRunUploaded(t *testing.T) { + apps := mariadbapp.GroupName + started := cleanupCreated.Add(time.Minute) + theirs := s3ObjectInfo{Key: dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip"), Size: 42, LastModified: started.Add(time.Minute)} + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "tenant/db" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "db", APIGroup: &apps}, + }, + } + other := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-other" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + + prober := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{theirs}}, job, other(), s3CredsSecret("tenant", "creds")) + created, err := prober.createMariaDBBackupArtifact(context.Background(), job, resolved, cleanupOperatorBackup(t), rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if got, want := created.Spec.DriverMetadata[mariadbDumpFileKey], strings.TrimPrefix(theirs.Key, cleanupPrefix); got != want { + t.Fatalf("probe recorded dump %q, want %q: this test covers the recorded path", got, want) + } + + backup := created.DeepCopy() + backup.ResourceVersion = "" + now := metav1.Now() + backup.DeletionTimestamp = &now + backup.Finalizers = []string{backupFinalizer} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{theirs}} + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), other(), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) { + t.Run("operator Backup is gone", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } + }) + t.Run("Backup records no operator Backup", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + delete(backup.Spec.DriverMetadata, mariadbBackupNameKey) + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup's Job failed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup has no Complete condition", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup never completed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Status = metav1.ConditionFalse + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("dump is on a volume", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage = mariadbtypes.BackupStorage{} + r, rec := cleanupReconciler(t, bucket, op, tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) +} + +func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"} + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob { + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}}, + Status: backupsv1alpha1.RestoreJobStatus{Phase: phase}, + } +} + +// operatorRestoreOf is a k8s.mariadb.com/Restore reading the dump of the +// operator Backup backupRef, with the given Complete condition. +func operatorRestoreOf(name, backupRef string, complete *metav1.Condition) *mariadbtypes.Restore { + restore := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}} + if backupRef != "" { + restore.Spec.BackupRef = &mariadbtypes.BackupReference{Name: backupRef} + } + if complete != nil { + restore.Status.Conditions = []metav1.Condition{*complete} + } + return restore +} + +func restoreComplete(status metav1.ConditionStatus, reason string) *metav1.Condition { + return &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: reason, LastTransitionTime: metav1.Now()} +} + +// The RestoreJob's phase is not the restore: one failed on its deadline +// leaves its operator Restore running, and that Restore still reads the +// dump. Until the operator marks it Complete, as it does a finished or a +// failed Job, the dump stays. +func TestCleanupMariaDBBackup_WaitsForAnOperatorRestoreOfItsDump(t *testing.T) { + held := map[string]*metav1.Condition{ + "running": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobRunning), + "suspended": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobSuspended), + "not yet reconciled": nil, + } + for name, cond := range held { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseFailed), operatorRestoreOf("rj-abcde", "op-bk", cond)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the operator Restore runs", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), "k8s.mariadb.com/Restore rj-abcde") { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the operator Restore", events) + } + }) + } + released := map[string]*mariadbtypes.Restore{ + "completed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobComplete)), + "failed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobFailed)), + "of another operator Backup": operatorRestoreOf("rj-abcde", "op-other", nil), + "from storage, not a Backup": operatorRestoreOf("rj-abcde", "", nil), + } + for name, restore := range released { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), restore) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) + } + t.Run("one the cache has not seen yet", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, operatorRestoreOf("rj-abcde", "op-bk", nil)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) +} + +func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { + for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { + t.Run(string(phase), func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", phase)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events) + } + }) + } + // A restore checks that its Backup is not being deleted before it + // starts, so one that started first is on the API server before the + // delete, though not necessarily in this controller's cache yet. + t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) + t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded), + restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed), + restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) +} + +// platformCredsConfig points the reconciler at the platform's credentials +// Secret, the source the per-tenant Secret is projected from. The projected +// Secret is "creds", the one cleanupOperatorBackup names, as the default +// strategy names cozy-backups-creds. +func platformCredsConfig(endpoint string) BackupCredentialsConfig { + return BackupCredentialsConfig{ + SourceNamespace: "cozy-system", + SourceSecretName: "cozy-backups-credentials", + TargetSecretName: "creds", + Endpoint: endpoint, + Region: "us-east-1", + } +} + +func platformCredsSecret(bucketName, endpoint string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"}, + Data: map[string][]byte{ + "accessKey": []byte("PLATFORM"), + "secretKey": []byte("platform-secret"), + "endpoint": []byte(endpoint), + "bucketName": []byte(bucketName), + }, + } +} + +func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" { + t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn) + } + } +} + +func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("another-bucket", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn) + } + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + for _, bucketName := range []string{"bkt", "tenant-own-bucket"} { + t.Run(bucketName, func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.Bucket = bucketName + // No platform Secret: the projection source is missing. + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } + }) + } +} + +// A missing or malformed platform Secret is a standing state, and the +// operator Backup's own credentials are the best left. A failed read says +// nothing about the Secret: falling back then can list a moved endpoint's +// old address, find no dump there and report it deleted. +func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}} + r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" { + t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("http://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed) + } +} + +func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("https://s3.configured.example") + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil { + t.Fatal(err) + } + if bucket.conns[0].Endpoint != "s3.configured.example" { + t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0]) + } +} + +// blockingBucket lists the dump it holds and answers the call named by +// blockOn only when its context ends, as an unreachable endpoint does. +type blockingBucket struct { + fakeS3Bucket + blockOn string + entered chan struct{} +} + +func (b *blockingBucket) block(ctx context.Context, call string) bool { + if b.blockOn != call { + return false + } + close(b.entered) + <-ctx.Done() + return true +} + +func (b *blockingBucket) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + if b.block(ctx, "list") { + return nil, ctx.Err() + } + return b.fakeS3Bucket.ListObjects(ctx, bucket, prefix, recursive) +} + +func (b *blockingBucket) RemoveObject(ctx context.Context, bucket, key string) error { + if b.block(ctx, "remove") { + return ctx.Err() + } + return b.fakeS3Bucket.RemoveObject(ctx, bucket, key) +} + +func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { + prev := mariadbDumpCleanupTimeout + mariadbDumpCleanupTimeout = 50 * time.Millisecond + t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) + for _, call := range []string{"list", "remove"} { + t.Run(call, func(t *testing.T) { + bucket := &blockingBucket{blockOn: call, entered: make(chan struct{})} + bucket.objects = []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectLister: func(s3Connection) (s3ObjectLister, error) { return bucket, nil }, + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + select { + case <-bucket.entered: + default: + t.Fatalf("cleanup returned without reaching the blocking %s", call) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatalf("cleanup did not give up on an unreachable endpoint blocking %s", call) + } + }) + } +} + +// Without the mariadb-operator CRDs every k8s.mariadb.com read fails, the +// listing of operator Restores included: no Restore can run then, so the +// Backup is released rather than held for good. +func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()} + } + return c.Get(ctx, key, obj, opts...) + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Restore").GroupKind()} + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName()) + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } +} + +// A strategy of its own that happens to use the platform bucket's name, as +// an admin-chosen name on external S3 can, is reached as recorded: only the +// projected Secret marks a Backup as written to the platform bucket. +func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds" + op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds" + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } +} + +// With no endpoint anywhere the platform credentials are unusable, which +// falls back like any other unreadable credentials. +func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + malformed := platformCredsSecret("bkt", "s3.new.example") + delete(malformed.Data, "secretKey") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// Reconcile keeps the finalizer while the dump cannot be deleted, and drops +// it once both the dump and the operator Backup are gone. +func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503")} + backup := cleanupBackup(cleanupDumpName) + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil { + t.Fatalf("Backup gone while its dump could not be deleted: %v", err) + } + + bucket.removeErr = nil + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } +} + +// The minio adapter deletes the exact key, with the colons of the dump's +// timestamp escaped in the path; SeaweedFS refuses them raw. +func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) { + var method, rawPath, auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil { + t.Fatalf("RemoveObject: %v", err) + } + if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" { + t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath) + } + if !strings.Contains(auth, "Credential=AKID/") { + t.Errorf("request not signed with the access key: %q", auth) + } +} + +func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil { + t.Fatal("RemoveObject = nil, want the AccessDenied error") + } +} + +func (f *fakeS3Bucket) StatObject(_ context.Context, _, key string) (int64, error) { + for _, o := range f.objects { + if o.Key == key { + return o.Size, nil + } + } + return 0, errors.New("not found") +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 452fff071..a8fd29a01 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -46,6 +47,15 @@ const ( mariadbBackupNamespaceKey = "k8s.mariadb.com/backup-namespace" mariadbStorageURIKey = "k8s.mariadb.com/storage-uri" + // The dump the operator Backup wrote: its file name under the storage + // prefix, and the RFC3339 timestamp in that name. Every run of an + // application writes under the same prefix, so only this name tells its + // dump from the others, and a restore asks for exactly that dump. + // Recorded only when findMariaDBDump proved the dump belongs to the + // operator Backup. + mariadbDumpFileKey = "k8s.mariadb.com/dump-file" + mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" + // Polling cadence for the operator Backup/Restore lifecycle. mariadbPollInterval = 5 * time.Second @@ -59,6 +69,12 @@ const ( // to terminate. Tenants override via spec.options.restoreTimeoutSeconds. mariadbDefaultRestoreDeadline = 30 * time.Minute + // Retry cadence while the dump a restore asks for cannot be checked. + // Longer than mariadbPollInterval because each attempt can hold the + // RestoreJob controller's single worker for + // mariadbRestoreDumpCheckTimeout. + mariadbDumpCheckRetryInterval = 30 * time.Second + // mariadbBackupSnapshotKind is the Kind stamped onto the snapshot // persisted in Backup.status.underlyingResources. The same shape carries // the storage descriptor and the rendered strategy parameters used at @@ -72,6 +88,12 @@ const ( // existing API surface. var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String() +// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a +// restore makes before it creates the operator Restore. The RestoreJob +// controller has a single worker, so an unreachable endpoint stalls every +// other RestoreJob for this long. A variable so tests can shorten it. +var mariadbRestoreDumpCheckTimeout = 10 * time.Second + // mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a // cozystack MariaDB application instance. The mapping mirrors the mariadb // ApplicationDefinition (release.prefix=mariadb-). @@ -420,7 +442,11 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( if uri := driverMD[mariadbStorageURIKey]; uri != "" { status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} if probe { - status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time) + if dump, ok := r.attributedMariaDBDump(ctx, mdbBackup, takenAt.Time); ok { + status.Artifact.SizeBytes = dump.Size + driverMD[mariadbDumpFileKey] = dump.Name + driverMD[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + } } } backup := &backupsv1alpha1.Backup{ @@ -592,9 +618,26 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest return ctrl.Result{}, err } - mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName) + mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + // Retried: failing the RestoreJob here could fail it under an + // operator Restore that is running. + return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err) + } + // The dump is chosen once, as the operator Restore is created; one that + // already exists has made its choice. + if mdbRestore == nil { + targetTime, err := r.mariadbRestoreTargetTime(ctx, restoreJob, backup, srcBackup) + var refused mariadbRestoreRefused + if errors.As(err, &refused) { + return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) + } + if err != nil { + return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err) + } + if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + } } cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) @@ -628,12 +671,114 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } } -// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with -// the RestoreJob, or returns the existing one if a previous reconcile -// already created it. -func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) { +// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be +// restored. It fails the RestoreJob, where any other error from choosing +// the dump is retried. +type mariadbRestoreRefused string + +func (e mariadbRestoreRefused) Error() string { return string(e) } + +// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator +// Restore for backup: the timestamp in the name of the dump backup +// records. mariadb-operator restores the dump whose timestamp equals the +// target and, when there is none, the nearest one, so the dump is first +// checked to still exist. A Backup that records no dump is resolved by +// legacyMariaDBRestoreTargetTime. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // Deleting a Backup deletes its dump, and the operator restores the + // nearest other dump once its own is gone, so a restore must not start + // from a Backup being deleted. The copy in hand can predate the delete; + // the cleanup holds the Backup only for restores that exist when it + // lists them. + live := &backupsv1alpha1.Backup{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { + if apierrors.IsNotFound(err) { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name)) + } + return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err) + } + if !live.DeletionTimestamp.IsZero() { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name)) + } + // A failed Job is Complete=True too, with reason JobFailed, and a + // BackupJob that took Complete=True for success made a Backup of it. + if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { + return "", mariadbRestoreRefused(fmt.Sprintf( + "the backup run of k8s.mariadb.com/Backup %s/%s failed (Complete=%s/%s) and left no dump to restore; restoring would bring back another run's dump", + src.Namespace, src.Name, cond.Status, cond.Reason)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name == "" { + return r.legacyMariaDBRestoreTargetTime(ctx, rj, backup, src), nil + } + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) + } + exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) + if err != nil { + return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) + } + if !exists { + s3 := src.Spec.Storage.S3 + return "", mariadbRestoreRefused(fmt.Sprintf( + "dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump", + name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix))) + } + return ts.UTC().Format(time.RFC3339), nil +} + +// legacyMariaDBRestoreTargetTime chooses the dump of a Backup taken before +// Backups recorded theirs: the one findMariaDBDump proves belongs to src, +// with the completion time the size probe was given, which takenAt keeps. +// When none is proven, for any reason, it returns empty, and the operator +// restores the newest dump under the prefix as such a restore always did: +// refusing would leave these Backups unrestorable, and the controller may +// never have been able to reach their storage. A Warning event says the +// dump is a best effort. +func (r *RestoreJobReconciler) legacyMariaDBRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) string { + dump, err := findMariaDBDump(ctx, r.objectStore(), src, backup.Spec.TakenAt.Time) + if err != nil { + getLogger(ctx).Info("no dump proven for a MariaDB Backup that records none; restoring the newest", "restorejob", rj.Name, "backup", backup.Name, "error", err) + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "BestEffortDump", + "Backup %s records no dump and none in storage could be proven to belong to k8s.mariadb.com/Backup %s/%s (%v); mariadb-operator restores the newest dump under its prefix as a best effort, which may be another backup run's", + backup.Name, src.Namespace, src.Name, err) + return "" + } + return dump.Timestamp.UTC().Format(time.RFC3339) +} + +// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False +// saying why, while the dump it asks for cannot be checked, and fails it +// with the cause once the restore deadline has passed. +func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) { + deadline := options.effectiveRestoreDeadline() + if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf( + "%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline)) + } + getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause) + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "DumpCheckPending", + Message: cause.Error(), + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil +} + +// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled +// with the RestoreJob, if a previous reconcile already created one. It is +// read uncached: one the cache has not caught up with would be created +// again, or, once its Backup is being deleted, refused while it runs. +func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} - if err := r.List(ctx, list, + if err := r.apiReader().List(ctx, list, client.InNamespace(rj.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: rj.Name, @@ -642,10 +787,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac ); err != nil { return nil, err } - if len(list.Items) > 0 { - return &list.Items[0], nil + if len(list.Items) == 0 { + return nil, nil } + return &list.Items[0], nil +} +// createMariaDBRestore creates the k8s.mariadb.com/Restore of the +// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty. +func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) { obj := &mariadbtypes.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: rj.Namespace, @@ -661,6 +811,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac Kind: mariadbtypes.BackupKind, Name: sourceBackupName, }, + TargetRecoveryTime: targetRecoveryTime, }, } if err := r.Create(ctx, obj); err != nil { diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index f03b67d94..9696d1eac 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -317,7 +317,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) { // Restore-side ensure idempotency + target rewrite // --------------------------------------------------------------------------- -func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { +func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()} rj := newMariaDBRestoreJob("rj-1", "tenant") @@ -325,16 +325,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { t.Fatalf("seed RestoreJob: %v", err) } - first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") - if err != nil { - t.Fatalf("first ensureMariaDBRestore: %v", err) + if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil { + t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err) } - second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") + created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z") if err != nil { - t.Fatalf("second ensureMariaDBRestore: %v", err) + t.Fatalf("createMariaDBRestore: %v", err) } - if first.Name != second.Name { - t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) + found, err := r.findMariaDBRestoreForJob(context.Background(), rj) + if err != nil || found == nil || found.Name != created.Name { + t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err) } list := &mariadbtypes.RestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { @@ -350,6 +350,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind { t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef) } + if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" { + t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime) + } } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a4dd7943..9737eabdc 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -57,6 +57,11 @@ type RestoreJobReconciler struct { // see a peer's StartedAt before the cache does. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to check that a + // backup's object still exists; nil means a real minio client. Set by + // unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 735e16308..883fd1fe9 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -164,15 +164,16 @@ rules: # MariaDB strategy: read the operator-side MariaDB CR to surface "not yet # rendered" preconditions, create one-shot k8s.mariadb.com/Backup CRs on # the BackupJob path, and create k8s.mariadb.com/Restore CRs on the -# RestoreJob path. The driver only ever Creates/Lists/Gets these objects - -# it does not patch or delete them; their lifecycle is owned by -# mariadb-operator and (for cleanup) by tenants / explicit teardown flows, -# never by this controller. +# RestoreJob path. A Backup is deleted once the dump it wrote is, when its +# Cozystack Backup is deleted; a Restore is never deleted. - apiGroups: ["k8s.mariadb.com"] resources: ["mariadbs"] verbs: ["get", "list", "watch"] - apiGroups: ["k8s.mariadb.com"] - resources: ["backups", "restores"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] verbs: ["get", "list", "watch", "create"] # MongoDB strategy: read the operator-side PerconaServerMongoDB CR to gate on # the backup preconditions, create one-shot PerconaServerMongoDBBackup CRs on the diff --git a/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml new file mode 100644 index 000000000..21ac6e807 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml @@ -0,0 +1,28 @@ +suite: RBAC of the MariaDB dump cleanup + +# Deleting a MariaDB Backup deletes its operator Backup once the dump is gone. +# The Go unit tests run with no authorization, so a missing verb only shows up +# on a cluster, where every such Backup would stay Terminating. + +templates: + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "ClusterRole can delete operator Backups but not Restores" + asserts: + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] + verbs: ["get", "list", "watch", "create"]