diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 1919d41c9..5ec96e663 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -260,6 +260,28 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { t.Errorf("created %d operator Restores; want the existing one followed", len(got)) } }) + // A failed lookup says nothing about whether the operator Restore + // exists, so the RestoreJob must not be failed under one that runs. + t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil { + t.Error("err = nil, want the failed lookup returned for a retry") + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores without knowing whether one exists", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index b97f7548b..28a24a14e 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -598,7 +598,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + // Retried: failing the RestoreJob here could fail it under an + // operator Restore that is running. + return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err) } // The dump is chosen once, as the operator Restore is created; one that // already exists has made its choice.