From 654c543264bbcba9c939a7c8b436ffbc02340c4d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:04:53 +0300 Subject: [PATCH] fix(backups): see a MariaDB restore the cache has not caught up with A MariaDB restore now checks on the API server that its Backup is not being deleted before it starts, so a restore that started first exists on the API server before the delete. The cleanup read RestoreJobs from the informer cache, which can lag behind, and could delete the dump under a restore it had not seen yet. Reading them uncached closes that window from both sides. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 15 +++++++++++++++ .../backupcontroller/mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 9d755059e..d8c69bf1c 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -43,6 +43,10 @@ type BackupReconciler struct { // cleanups project them into the Backup's namespace, and the MariaDB // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + // APIReader is the manager's uncached reader. The MariaDB cleanup lists + // the RestoreJobs of a Backup through it: a restore that started before + // the delete may not be in the cache yet. Wired in SetupWithManager. + APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the // MariaDB cleanup deletes a dump with; nil means a real minio client. @@ -294,7 +298,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.APIReader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) } + +// apiReader returns the uncached reader. Reconcilers built by hand in unit +// tests have none and fall back to their fake Client, which has no cache to +// bypass. +func (r *BackupReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 94a7d10f4..531b806c9 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -200,7 +200,7 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { list := &backupsv1alpha1.RestoreJobList{} - if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) } for i := range list.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ddca34523..20f2757d4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -763,6 +763,18 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { } }) } + // A restore checks that its Backup is not being deleted before it + // starts, so one that started first is on the API server before the + // delete, though not necessarily in this controller's cache yet. + t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),