fix(backups): read the runs that may own a MariaDB dump uncached

Before deleting a dump the cleanup checks every other operator Backup
on the same prefix that could have written it, and it read them from
the informer cache. A run whose operator Backup had not reached the
cache yet but had already uploaded its dump was missed, and its dump
deleted as this Backup's. Both the recorded and the legacy paths now
list the runs from the API server, as the restore checks already do.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-12 17:01:29 +03:00
1 parent a6a704768c
commit b3ca798bc4
3 files changed
+33 -5

No files matched your search

@@ -44,9 +44,10 @@ type BackupReconciler struct {
// cleanup reads them to reach the platform bucket.
CredentialsConfig BackupCredentialsConfig
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
// the RestoreJobs and operator Restores of a Backup through it: a
// restore that started before the delete may not be in the cache yet.
// Wired in SetupWithManager.
// the RestoreJobs and operator Restores of a Backup, and the operator
// Backups that may own its dump, through it: a restore that started
// before the delete, or a run that has already uploaded its dump, may
// not be in the cache yet. Wired in SetupWithManager.
APIReader client.Reader
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
@@ -120,7 +120,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac
}
absent := false
if name != "" {
other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name)
other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name)
absent = !found
if err != nil {
return r.retryMariaDBCleanup(ctx, backup, location+name, err)
@@ -130,7 +130,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac
fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil
}
} else {
dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true)
dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true)
if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) {
return r.releaseMariaDBCleanup(ctx, backup, location,
fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil
@@ -547,6 +547,33 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T
}
}
// The runs that may own a dump are read from the API server: a run whose
// operator Backup has not reached the cache yet can already have uploaded
// its dump, and missing it would delete that dump as this Backup's.
func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) {
started := cleanupCreated.Add(time.Minute)
theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip")
unseen := func() *mariadbtypes.Backup {
b := cleanupOperatorBackup(t)
b.Name = "op-unseen"
b.CreationTimestamp = metav1.NewTime(started)
b.Status.Conditions = nil
return b
}
cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""}
for name, recorded := range cases {
t.Run(name, func(t *testing.T) {
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}}
backup := cleanupBackup(recorded)
r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen())
res, err := r.cleanupMariaDBBackup(context.Background(), backup)
assertReleasedUntouched(t, r, rec, bucket, res, err)
})
}
}
// A run started after the recorded dump was uploaded cannot have written it.
func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) {
key := cleanupPrefix + cleanupDumpName