fix(backups): see a MariaDB restore the cache has not caught up with

A MariaDB restore now checks on the API server that its Backup is not
being deleted before it starts, so a restore that started first exists
on the API server before the delete. The cleanup read RestoreJobs from
the informer cache, which can lag behind, and could delete the dump
under a restore it had not seen yet. Reading them uncached closes that
window from both sides.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-11 15:04:53 +03:00
1 parent 320d0299ad
commit 654c543264
3 files changed
+28 -1

No files matched your search

@@ -43,6 +43,10 @@ type BackupReconciler struct {
// cleanups project them into the Backup's namespace, and the MariaDB
// cleanup reads them to reach the platform bucket.
CredentialsConfig BackupCredentialsConfig
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
// the RestoreJobs of a Backup through it: a restore that started before
// the delete may not be in the cache yet. Wired in SetupWithManager.
APIReader client.Reader
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
// MariaDB cleanup deletes a dump with; nil means a real minio client.
@@ -294,7 +298,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup
// SetupWithManager registers the BackupReconciler with the Manager.
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.APIReader = mgr.GetAPIReader()
return ctrl.NewControllerManagedBy(mgr).
For(&backupsv1alpha1.Backup{}).
Complete(r)
}
// apiReader returns the uncached reader. Reconcilers built by hand in unit
// tests have none and fall back to their fake Client, which has no cache to
// bypass.
func (r *BackupReconciler) apiReader() client.Reader {
if r.APIReader != nil {
return r.APIReader
}
return r.Client
}
@@ -200,7 +200,7 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli
// backup and has not finished, or "".
func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) {
list := &backupsv1alpha1.RestoreJobList{}
if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil {
if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil {
return "", fmt.Errorf("list RestoreJobs: %w", err)
}
for i := range list.Items {
@@ -763,6 +763,18 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) {
}
})
}
// A restore checks that its Backup is not being deleted before it
// starts, so one that started first is on the API server before the
// delete, though not necessarily in this controller's cache yet.
t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) {
bucket := &fakeS3Bucket{}
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning))
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
if err != nil || !requeues(res) || len(bucket.removed) != 0 {
t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed)
}
})
t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) {
bucket := &fakeS3Bucket{}
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),