mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(mariadb): restore the dump a Backup records
A RestoreJob created a mariadb-operator Restore that named only the operator Backup. The operator then lists the prefix that Backup wrote to and restores the dump nearest to the moment the restore started: the application's newest dump, not the one the Backup was taken as. When the Backup records its dump, the Restore now carries the timestamp in that dump's name as targetRecoveryTime. mariadb-operator returns the dump whose timestamp equals the target before it looks for the nearest one, and the nearest one is exactly what it would restore if the recorded dump were gone. So the controller first checks, with its own bounded call, that the dump is still in the operator Backup's storage. A dump that is gone fails the RestoreJob, naming the dump, instead of restoring a neighbour in its place. A check that cannot be made keeps the RestoreJob waiting with Ready=False until the restore deadline, then fails it with the cause: the Backup could record its dump only because the controller reached that storage when it was taken, so an unreachable storage is not a reason to guess. The dump is chosen once, as the operator Restore is created; a Restore that already exists is followed as before. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
6 files changed
+467
-28
No files matched your search
@@ -90,6 +90,10 @@ func (r *BackupJobReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
|
||||
}
|
||||
|
||||
func (r *RestoreJobReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
return r.objectStore().lister(conn)
|
||||
}
|
||||
|
||||
@@ -60,22 +60,14 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack
|
||||
// from can have been edited since, and pointing at another bucket now says
|
||||
// nothing about where this dump went.
|
||||
func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) {
|
||||
s3 := mdbBackup.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return mariadbDump{}, errors.New("the operator Backup does not store to S3")
|
||||
lister, s3, err := mariadbStorage(ctx, store, mdbBackup)
|
||||
if err != nil {
|
||||
return mariadbDump{}, err
|
||||
}
|
||||
others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err)
|
||||
}
|
||||
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err)
|
||||
}
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("build S3 client: %w", err)
|
||||
}
|
||||
// Non-recursive, as mariadb-operator lists it: the default prefix is also
|
||||
// the default CNPG destinationPath, so an app of each kind with the same
|
||||
// name shares it and a recursive listing would walk the WAL archive.
|
||||
@@ -88,6 +80,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt
|
||||
return matchMariaDBDump(objects, prefix, own, others)
|
||||
}
|
||||
|
||||
// mariadbDumpExists reports whether the dump file name is still in the
|
||||
// storage mdbBackup wrote to.
|
||||
func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) {
|
||||
lister, s3, err := mariadbStorage(ctx, store, mdbBackup)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
key := mariadbOperatorS3Prefix(s3.Prefix) + name
|
||||
objects, err := lister.ListObjects(ctx, s3.Bucket, key, false)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err)
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.Key == key {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials
|
||||
// its own spec names.
|
||||
func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) {
|
||||
s3 := mdbBackup.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return nil, nil, errors.New("the operator Backup does not store to S3")
|
||||
}
|
||||
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err)
|
||||
}
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("build S3 client: %w", err)
|
||||
}
|
||||
return lister, s3, nil
|
||||
}
|
||||
|
||||
// mariadbRun is the span in which an operator Backup can have written its
|
||||
// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side.
|
||||
type mariadbRun struct {
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
// The source run of the restore fixture: operator Backup op-bk of
|
||||
// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing
|
||||
// under tenant/src/ with mdbRestoreDump as its dump.
|
||||
var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
const (
|
||||
mdbRestorePrefix = "tenant/src/"
|
||||
mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql"
|
||||
mdbRestoreTime = "2026-10-21T10:00:05Z"
|
||||
mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql"
|
||||
)
|
||||
|
||||
type mariadbRestoreFixture struct {
|
||||
c client.Client
|
||||
r *RestoreJobReconciler
|
||||
rj *backupsv1alpha1.RestoreJob
|
||||
backup *backupsv1alpha1.Backup
|
||||
src *mariadbtypes.Backup
|
||||
lister *fakeS3Lister
|
||||
events *record.FakeRecorder
|
||||
}
|
||||
|
||||
// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose
|
||||
// driverMetadata is driverMD plus the operator Backup's name, into the
|
||||
// existing application target. extra objects are seeded too.
|
||||
func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture {
|
||||
t.Helper()
|
||||
apps := mariadbapp.GroupName
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Prefix = "tenant/src"
|
||||
src := operatorBackupStoringTo(t, s3, mdbRestoreT0)
|
||||
src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))}
|
||||
|
||||
md := map[string]string{mariadbBackupNameKey: src.Name}
|
||||
for k, v := range driverMD {
|
||||
md[k] = v
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)),
|
||||
DriverMetadata: md,
|
||||
},
|
||||
}
|
||||
started := metav1.Now()
|
||||
rj := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{
|
||||
BackupRef: corev1.LocalObjectReference{Name: backup.Name},
|
||||
TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps},
|
||||
},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
||||
}
|
||||
target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}}
|
||||
objs := append([]client.Object{src, backup, rj, target}, extra...)
|
||||
c := newMariaDBStrategyTestClient(t, objs...)
|
||||
fetched := &backupsv1alpha1.RestoreJob{}
|
||||
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
events := record.NewFakeRecorder(10)
|
||||
return &mariadbRestoreFixture{
|
||||
c: c,
|
||||
rj: fetched,
|
||||
backup: backup,
|
||||
src: src,
|
||||
lister: lister,
|
||||
events: events,
|
||||
r: &RestoreJobReconciler{
|
||||
Client: c,
|
||||
Recorder: events,
|
||||
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
|
||||
lister.conn = conn
|
||||
return lister, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result {
|
||||
t.Helper()
|
||||
res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup)
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileMariaDBRestore: %v", err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore {
|
||||
t.Helper()
|
||||
list := &mariadbtypes.RestoreList{}
|
||||
if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil {
|
||||
t.Fatalf("list operator Restores: %v", err)
|
||||
}
|
||||
return list.Items
|
||||
}
|
||||
|
||||
// wantFailed asserts the RestoreJob failed with a message holding every
|
||||
// fragment, and that no operator Restore was created.
|
||||
func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) {
|
||||
t.Helper()
|
||||
if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase)
|
||||
}
|
||||
for _, s := range fragments {
|
||||
if !strings.Contains(f.rj.Status.Message, s) {
|
||||
t.Errorf("message %q does not contain %q", f.rj.Status.Message, s)
|
||||
}
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores; want none", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// wantWaiting asserts the RestoreJob is kept waiting on the dump check
|
||||
// with Ready=False saying why, and that no operator Restore was created.
|
||||
func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) {
|
||||
t.Helper()
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message)
|
||||
}
|
||||
if res.RequeueAfter != mariadbDumpCheckRetryInterval {
|
||||
t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval)
|
||||
}
|
||||
persisted := &backupsv1alpha1.RestoreJob{}
|
||||
if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready")
|
||||
if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" {
|
||||
t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond)
|
||||
}
|
||||
for _, s := range fragments {
|
||||
if !strings.Contains(cond.Message, s) {
|
||||
t.Errorf("Ready message %q does not contain %q", cond.Message, s)
|
||||
}
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores; want none", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// wantRestore asserts exactly one operator Restore of op-bk into target,
|
||||
// asking for targetTime.
|
||||
func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) {
|
||||
t.Helper()
|
||||
got := f.restores(t)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("created %d operator Restores; want one", len(got))
|
||||
}
|
||||
spec := got[0].Spec
|
||||
if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") {
|
||||
t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target"))
|
||||
}
|
||||
if spec.TargetRecoveryTime != targetTime {
|
||||
t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime)
|
||||
}
|
||||
}
|
||||
|
||||
func recordedDump(name string) map[string]string {
|
||||
return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime}
|
||||
}
|
||||
|
||||
func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
|
||||
t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)},
|
||||
{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)},
|
||||
}}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds)
|
||||
f.reconcile(t)
|
||||
f.wantRestore(t, mdbRestoreTime)
|
||||
if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" {
|
||||
t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds)
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix)
|
||||
})
|
||||
|
||||
t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds)
|
||||
f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.")
|
||||
})
|
||||
|
||||
t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) {
|
||||
l := &fakeS3Lister{}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l)
|
||||
f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials")
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times without credentials", l.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds)
|
||||
f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
var res ctrl.Result
|
||||
var err error
|
||||
withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) })
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileMariaDBRestore: %v", err)
|
||||
}
|
||||
f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error())
|
||||
})
|
||||
|
||||
t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds)
|
||||
long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute))
|
||||
f.rj.Status.StartedAt = &long
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after")
|
||||
})
|
||||
|
||||
t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) {
|
||||
l := &fakeS3Lister{}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds)
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, `"dump.sql"`)
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a name that is not a dump", l.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) {
|
||||
l := &fakeS3Lister{err: errors.New("Access Denied.")}
|
||||
existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "rj-abcde",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: "rj",
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
||||
},
|
||||
}}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing)
|
||||
res := f.reconcile(t)
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls)
|
||||
}
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval {
|
||||
t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter)
|
||||
}
|
||||
if got := f.restores(t); len(got) != 1 {
|
||||
t.Errorf("%d operator Restores; want only the existing one", len(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) {
|
||||
l := &fakeS3Lister{err: errors.New("Access Denied.")}
|
||||
f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds"))
|
||||
f.reconcile(t)
|
||||
f.wantRestore(t, "")
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// withShortDumpCheckTimeout shortens the restore's dump check timeout and
|
||||
// fails the test if fn is not back well before the default would expire.
|
||||
func withShortDumpCheckTimeout(t *testing.T, fn func()) {
|
||||
t.Helper()
|
||||
saved := mariadbRestoreDumpCheckTimeout
|
||||
mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond
|
||||
defer func() { mariadbRestoreDumpCheckTimeout = saved }()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
fn()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the dump check did not give up at its timeout")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package backupcontroller
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -50,8 +51,9 @@ const (
|
||||
// The dump the operator Backup wrote: its file name under the storage
|
||||
// prefix, and the RFC3339 timestamp in that name. Every run of an
|
||||
// application writes under the same prefix, so only this name tells its
|
||||
// dump from the others. Recorded only when findMariaDBDump proved the
|
||||
// dump belongs to the operator Backup.
|
||||
// dump from the others, and a restore asks for exactly that dump.
|
||||
// Recorded only when findMariaDBDump proved the dump belongs to the
|
||||
// operator Backup.
|
||||
mariadbDumpFileKey = "k8s.mariadb.com/dump-file"
|
||||
mariadbDumpTimeKey = "k8s.mariadb.com/dump-time"
|
||||
|
||||
@@ -68,6 +70,12 @@ const (
|
||||
// to terminate. Tenants override via spec.options.restoreTimeoutSeconds.
|
||||
mariadbDefaultRestoreDeadline = 30 * time.Minute
|
||||
|
||||
// Retry cadence while the dump a restore asks for cannot be checked.
|
||||
// Longer than mariadbPollInterval because each attempt can hold the
|
||||
// RestoreJob controller's single worker for
|
||||
// mariadbRestoreDumpCheckTimeout.
|
||||
mariadbDumpCheckRetryInterval = 30 * time.Second
|
||||
|
||||
// mariadbBackupSnapshotKind is the Kind stamped onto the snapshot
|
||||
// persisted in Backup.status.underlyingResources. The same shape carries
|
||||
// the storage descriptor and the rendered strategy parameters used at
|
||||
@@ -81,6 +89,12 @@ const (
|
||||
// existing API surface.
|
||||
var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String()
|
||||
|
||||
// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a
|
||||
// restore makes before it creates the operator Restore. The RestoreJob
|
||||
// controller has a single worker, so an unreachable endpoint stalls every
|
||||
// other RestoreJob for this long. A variable so tests can shorten it.
|
||||
var mariadbRestoreDumpCheckTimeout = 10 * time.Second
|
||||
|
||||
// mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a
|
||||
// cozystack MariaDB application instance. The mapping mirrors the mariadb
|
||||
// ApplicationDefinition (release.prefix=mariadb-).
|
||||
@@ -582,10 +596,25 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName)
|
||||
mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob)
|
||||
if err != nil {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err))
|
||||
}
|
||||
// The dump is chosen once, as the operator Restore is created; one that
|
||||
// already exists has made its choice.
|
||||
if mdbRestore == nil {
|
||||
targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup)
|
||||
var refused mariadbRestoreRefused
|
||||
if errors.As(err, &refused) {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, refused.Error())
|
||||
}
|
||||
if err != nil {
|
||||
return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err)
|
||||
}
|
||||
if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete)
|
||||
switch {
|
||||
@@ -626,10 +655,68 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
}
|
||||
}
|
||||
|
||||
// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with
|
||||
// the RestoreJob, or returns the existing one if a previous reconcile
|
||||
// already created it.
|
||||
func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) {
|
||||
// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be
|
||||
// restored. It fails the RestoreJob, where any other error from choosing
|
||||
// the dump is retried.
|
||||
type mariadbRestoreRefused string
|
||||
|
||||
func (e mariadbRestoreRefused) Error() string { return string(e) }
|
||||
|
||||
// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator
|
||||
// Restore for backup: the timestamp in the name of the dump backup
|
||||
// records. mariadb-operator restores the dump whose timestamp equals the
|
||||
// target and, when there is none, the nearest one, so the dump is first
|
||||
// checked to still exist. Empty when backup records no dump, which leaves
|
||||
// the operator to take the newest dump under the prefix.
|
||||
func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) {
|
||||
name := backup.Spec.DriverMetadata[mariadbDumpFileKey]
|
||||
if name == "" {
|
||||
return "", nil
|
||||
}
|
||||
ts, ok := parseMariaDBLogicalBackupFile(name)
|
||||
if !ok {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout)
|
||||
defer cancel()
|
||||
exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err)
|
||||
}
|
||||
if !exists {
|
||||
s3 := src.Spec.Storage.S3
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf(
|
||||
"dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump",
|
||||
name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix)))
|
||||
}
|
||||
return ts.UTC().Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False
|
||||
// saying why, while the dump it asks for cannot be checked, and fails it
|
||||
// with the cause once the restore deadline has passed.
|
||||
func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) {
|
||||
deadline := options.effectiveRestoreDeadline()
|
||||
if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline {
|
||||
return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf(
|
||||
"%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline))
|
||||
}
|
||||
getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause)
|
||||
apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "DumpCheckPending",
|
||||
Message: cause.Error(),
|
||||
})
|
||||
if err := r.Status().Update(ctx, rj); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil
|
||||
}
|
||||
|
||||
// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled
|
||||
// with the RestoreJob, if a previous reconcile already created one.
|
||||
func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) {
|
||||
list := &mariadbtypes.RestoreList{}
|
||||
if err := r.List(ctx, list,
|
||||
client.InNamespace(rj.Namespace),
|
||||
@@ -640,10 +727,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(list.Items) > 0 {
|
||||
return &list.Items[0], nil
|
||||
if len(list.Items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &list.Items[0], nil
|
||||
}
|
||||
|
||||
// createMariaDBRestore creates the k8s.mariadb.com/Restore of the
|
||||
// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty.
|
||||
func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) {
|
||||
obj := &mariadbtypes.Restore{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: rj.Namespace,
|
||||
@@ -659,6 +751,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac
|
||||
Kind: mariadbtypes.BackupKind,
|
||||
Name: sourceBackupName,
|
||||
},
|
||||
TargetRecoveryTime: targetRecoveryTime,
|
||||
},
|
||||
}
|
||||
if err := r.Create(ctx, obj); err != nil {
|
||||
|
||||
@@ -313,7 +313,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) {
|
||||
// Restore-side ensure idempotency + target rewrite
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) {
|
||||
func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) {
|
||||
c := newMariaDBStrategyTestClient(t)
|
||||
r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
rj := newMariaDBRestoreJob("rj-1", "tenant")
|
||||
@@ -321,16 +321,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) {
|
||||
t.Fatalf("seed RestoreJob: %v", err)
|
||||
}
|
||||
|
||||
first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target")
|
||||
if err != nil {
|
||||
t.Fatalf("first ensureMariaDBRestore: %v", err)
|
||||
if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil {
|
||||
t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err)
|
||||
}
|
||||
second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target")
|
||||
created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z")
|
||||
if err != nil {
|
||||
t.Fatalf("second ensureMariaDBRestore: %v", err)
|
||||
t.Fatalf("createMariaDBRestore: %v", err)
|
||||
}
|
||||
if first.Name != second.Name {
|
||||
t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name)
|
||||
found, err := r.findMariaDBRestoreForJob(context.Background(), rj)
|
||||
if err != nil || found == nil || found.Name != created.Name {
|
||||
t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err)
|
||||
}
|
||||
list := &mariadbtypes.RestoreList{}
|
||||
if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil {
|
||||
@@ -346,6 +346,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) {
|
||||
if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind {
|
||||
t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef)
|
||||
}
|
||||
if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" {
|
||||
t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -57,6 +57,11 @@ type RestoreJobReconciler struct {
|
||||
// see a peer's StartedAt before the cache does. Wired in
|
||||
// SetupWithManager.
|
||||
APIReader client.Reader
|
||||
|
||||
// newS3ObjectLister overrides the S3 client used to check that a
|
||||
// backup's object still exists; nil means a real minio client. Set by
|
||||
// unit tests.
|
||||
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
}
|
||||
|
||||
func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
|
||||
Reference in new issue
Block a user