diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 941b00323..9197f07be 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -90,6 +90,10 @@ func (r *BackupJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *RestoreJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 950303d78..a61ec4a43 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,22 +60,14 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - s3 := mdbBackup.Spec.Storage.S3 - if s3 == nil { - return mariadbDump{}, errors.New("the operator Backup does not store to S3") + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return mariadbDump{}, err } others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } - conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) - if err != nil { - return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) - } - lister, err := store.lister(conn) - if err != nil { - return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) - } // Non-recursive, as mariadb-operator lists it: the default prefix is also // the default CNPG destinationPath, so an app of each kind with the same // name shares it and a recursive listing would walk the WAL archive. @@ -88,6 +80,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt return matchMariaDBDump(objects, prefix, own, others) } +// mariadbDumpExists reports whether the dump file name is still in the +// storage mdbBackup wrote to. +func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) { + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return false, err + } + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + for _, obj := range objects { + if obj.Key == key { + return true, nil + } + } + return false, nil +} + +// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials +// its own spec names. +func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return nil, nil, errors.New("the operator Backup does not store to S3") + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) + if err != nil { + return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return nil, nil, fmt.Errorf("build S3 client: %w", err) + } + return lister, s3, nil +} + // mariadbRun is the span in which an operator Backup can have written its // dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. type mariadbRun struct { diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go new file mode 100644 index 000000000..0375def33 --- /dev/null +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// The source run of the restore fixture: operator Backup op-bk of +// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing +// under tenant/src/ with mdbRestoreDump as its dump. +var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + +const ( + mdbRestorePrefix = "tenant/src/" + mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql" + mdbRestoreTime = "2026-10-21T10:00:05Z" + mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql" +) + +type mariadbRestoreFixture struct { + c client.Client + r *RestoreJobReconciler + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup + src *mariadbtypes.Backup + lister *fakeS3Lister + events *record.FakeRecorder +} + +// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose +// driverMetadata is driverMD plus the operator Backup's name, into the +// existing application target. extra objects are seeded too. +func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture { + t.Helper() + apps := mariadbapp.GroupName + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/src" + src := operatorBackupStoringTo(t, s3, mdbRestoreT0) + src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))} + + md := map[string]string{mariadbBackupNameKey: src.Name} + for k, v := range driverMD { + md[k] = v + } + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)), + DriverMetadata: md, + }, + } + started := metav1.Now() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: backup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}} + objs := append([]client.Object{src, backup, rj, target}, extra...) + c := newMariaDBStrategyTestClient(t, objs...) + fetched := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + events := record.NewFakeRecorder(10) + return &mariadbRestoreFixture{ + c: c, + rj: fetched, + backup: backup, + src: src, + lister: lister, + events: events, + r: &RestoreJobReconciler{ + Client: c, + Recorder: events, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + lister.conn = conn + return lister, nil + }, + }, + } +} + +func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result { + t.Helper() + res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + return res +} + +func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore { + t.Helper() + list := &mariadbtypes.RestoreList{} + if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { + t.Fatalf("list operator Restores: %v", err) + } + return list.Items +} + +// wantFailed asserts the RestoreJob failed with a message holding every +// fragment, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) { + t.Helper() + if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase) + } + for _, s := range fragments { + if !strings.Contains(f.rj.Status.Message, s) { + t.Errorf("message %q does not contain %q", f.rj.Status.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantWaiting asserts the RestoreJob is kept waiting on the dump check +// with Ready=False saying why, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) { + t.Helper() + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message) + } + if res.RequeueAfter != mariadbDumpCheckRetryInterval { + t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval) + } + persisted := &backupsv1alpha1.RestoreJob{} + if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" { + t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond) + } + for _, s := range fragments { + if !strings.Contains(cond.Message, s) { + t.Errorf("Ready message %q does not contain %q", cond.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantRestore asserts exactly one operator Restore of op-bk into target, +// asking for targetTime. +func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) { + t.Helper() + got := f.restores(t) + if len(got) != 1 { + t.Fatalf("created %d operator Restores; want one", len(got)) + } + spec := got[0].Spec + if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") { + t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target")) + } + if spec.TargetRecoveryTime != targetTime { + t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime) + } +} + +func recordedDump(name string) map[string]string { + return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} +} + +func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + + t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}, + {Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" { + t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump) + } + }) + + t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix) + }) + + t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.") + }) + + t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials") + if l.calls != 0 { + t.Errorf("S3 listed %d times without credentials", l.calls) + } + }) + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var res ctrl.Result + var err error + withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error()) + }) + + t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute)) + f.rj.Status.StartedAt = &long + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after") + }) + + t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds) + f.reconcile(t) + f.wantFailed(t, `"dump.sql"`) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a name that is not a dump", l.calls) + } + }) + + t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing) + res := f.reconcile(t) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls) + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval { + t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter) + } + if got := f.restores(t); len(got) != 1 { + t.Errorf("%d operator Restores; want only the existing one", len(got)) + } + }) +} + +func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds")) + f.reconcile(t) + f.wantRestore(t, "") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls) + } +} + +// withShortDumpCheckTimeout shortens the restore's dump check timeout and +// fails the test if fn is not back well before the default would expire. +func withShortDumpCheckTimeout(t *testing.T, fn func()) { + t.Helper() + saved := mariadbRestoreDumpCheckTimeout + mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond + defer func() { mariadbRestoreDumpCheckTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the dump check did not give up at its timeout") + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 50c871db3..86b42d33f 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -50,8 +51,9 @@ const ( // The dump the operator Backup wrote: its file name under the storage // prefix, and the RFC3339 timestamp in that name. Every run of an // application writes under the same prefix, so only this name tells its - // dump from the others. Recorded only when findMariaDBDump proved the - // dump belongs to the operator Backup. + // dump from the others, and a restore asks for exactly that dump. + // Recorded only when findMariaDBDump proved the dump belongs to the + // operator Backup. mariadbDumpFileKey = "k8s.mariadb.com/dump-file" mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" @@ -68,6 +70,12 @@ const ( // to terminate. Tenants override via spec.options.restoreTimeoutSeconds. mariadbDefaultRestoreDeadline = 30 * time.Minute + // Retry cadence while the dump a restore asks for cannot be checked. + // Longer than mariadbPollInterval because each attempt can hold the + // RestoreJob controller's single worker for + // mariadbRestoreDumpCheckTimeout. + mariadbDumpCheckRetryInterval = 30 * time.Second + // mariadbBackupSnapshotKind is the Kind stamped onto the snapshot // persisted in Backup.status.underlyingResources. The same shape carries // the storage descriptor and the rendered strategy parameters used at @@ -81,6 +89,12 @@ const ( // existing API surface. var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String() +// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a +// restore makes before it creates the operator Restore. The RestoreJob +// controller has a single worker, so an unreachable endpoint stalls every +// other RestoreJob for this long. A variable so tests can shorten it. +var mariadbRestoreDumpCheckTimeout = 10 * time.Second + // mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a // cozystack MariaDB application instance. The mapping mirrors the mariadb // ApplicationDefinition (release.prefix=mariadb-). @@ -582,10 +596,25 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest return ctrl.Result{}, err } - mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName) + mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) } + // The dump is chosen once, as the operator Restore is created; one that + // already exists has made its choice. + if mdbRestore == nil { + targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup) + var refused mariadbRestoreRefused + if errors.As(err, &refused) { + return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) + } + if err != nil { + return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err) + } + if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + } + } cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { @@ -626,10 +655,68 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } } -// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with -// the RestoreJob, or returns the existing one if a previous reconcile -// already created it. -func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) { +// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be +// restored. It fails the RestoreJob, where any other error from choosing +// the dump is retried. +type mariadbRestoreRefused string + +func (e mariadbRestoreRefused) Error() string { return string(e) } + +// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator +// Restore for backup: the timestamp in the name of the dump backup +// records. mariadb-operator restores the dump whose timestamp equals the +// target and, when there is none, the nearest one, so the dump is first +// checked to still exist. Empty when backup records no dump, which leaves +// the operator to take the newest dump under the prefix. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name == "" { + return "", nil + } + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() + exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) + if err != nil { + return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) + } + if !exists { + s3 := src.Spec.Storage.S3 + return "", mariadbRestoreRefused(fmt.Sprintf( + "dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump", + name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix))) + } + return ts.UTC().Format(time.RFC3339), nil +} + +// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False +// saying why, while the dump it asks for cannot be checked, and fails it +// with the cause once the restore deadline has passed. +func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) { + deadline := options.effectiveRestoreDeadline() + if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf( + "%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline)) + } + getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause) + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "DumpCheckPending", + Message: cause.Error(), + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil +} + +// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled +// with the RestoreJob, if a previous reconcile already created one. +func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} if err := r.List(ctx, list, client.InNamespace(rj.Namespace), @@ -640,10 +727,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac ); err != nil { return nil, err } - if len(list.Items) > 0 { - return &list.Items[0], nil + if len(list.Items) == 0 { + return nil, nil } + return &list.Items[0], nil +} +// createMariaDBRestore creates the k8s.mariadb.com/Restore of the +// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty. +func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) { obj := &mariadbtypes.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: rj.Namespace, @@ -659,6 +751,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac Kind: mariadbtypes.BackupKind, Name: sourceBackupName, }, + TargetRecoveryTime: targetRecoveryTime, }, } if err := r.Create(ctx, obj); err != nil { diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index 0e4f63b5e..fa033fb22 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -313,7 +313,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) { // Restore-side ensure idempotency + target rewrite // --------------------------------------------------------------------------- -func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { +func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()} rj := newMariaDBRestoreJob("rj-1", "tenant") @@ -321,16 +321,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { t.Fatalf("seed RestoreJob: %v", err) } - first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") - if err != nil { - t.Fatalf("first ensureMariaDBRestore: %v", err) + if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil { + t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err) } - second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") + created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z") if err != nil { - t.Fatalf("second ensureMariaDBRestore: %v", err) + t.Fatalf("createMariaDBRestore: %v", err) } - if first.Name != second.Name { - t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) + found, err := r.findMariaDBRestoreForJob(context.Background(), rj) + if err != nil || found == nil || found.Name != created.Name { + t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err) } list := &mariadbtypes.RestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { @@ -346,6 +346,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind { t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef) } + if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" { + t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime) + } } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a61f0800..db7348923 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -57,6 +57,11 @@ type RestoreJobReconciler struct { // see a peer's StartedAt before the cache does. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to check that a + // backup's object still exists; nil means a real minio client. Set by + // unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {