diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index b67c640ab..5e6292635 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,9 +44,10 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs and operator Restores of a Backup through it: a - // restore that started before the delete may not be in the cache yet. - // Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup, and the operator + // Backups that may own its dump, through it: a restore that started + // before the delete, or a run that has already uploaded its dump, may + // not be in the cache yet. Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 3ff8adfbe..912bb0038 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -120,7 +120,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac } absent := false if name != "" { - other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name) absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) @@ -130,7 +130,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil } } else { - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 354bf83f3..1d2f10cbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -547,6 +547,33 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T } } +// The runs that may own a dump are read from the API server: a run whose +// operator Backup has not reached the cache yet can already have uploaded +// its dump, and missing it would delete that dump as this Backup's. +func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + unseen := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-unseen" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""} + for name, recorded := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + backup := cleanupBackup(recorded) + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + // A run started after the recorded dump was uploaded cannot have written it. func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { key := cleanupPrefix + cleanupDumpName