fix(backups): refuse a MariaDB restore from a Backup being deleted

Deleting a MariaDB Backup now deletes its dump, and mariadb-operator
restores the dump nearest the requested time when the exact one is
gone. A restore that starts while the Backup's cleanup runs can lose
its dump between the check and the operator's read, and then completes
on another run's data without any error.

The restore now reads the Backup from the API server before creating
the operator Restore and fails when it is being deleted or is gone. The
copy the reconcile was handed comes from the cache and can predate the
delete; a transient read error keeps the RestoreJob waiting like any
other failed dump check.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-11 15:03:59 +03:00
1 parent 74fbb34aca
commit 320d0299ad
2 files changed
+66

No files matched your search

@@ -14,6 +14,7 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp"
@@ -184,6 +185,57 @@ func recordedDump(name string) map[string]string {
return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime}
}
// A Backup being deleted is deleting its dump, so a restore that starts
// now can find it gone once the operator reads it, and the operator then
// restores the nearest other dump. The Backup is read from the API server,
// since the copy the reconcile was handed can predate the delete.
func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) {
creds := s3CredsSecret("tenant", "creds")
present := func() *fakeS3Lister {
return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}}
}
cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil}
for name, md := range cases {
t.Run(name+", being deleted", func(t *testing.T) {
f := newMariaDBRestoreFixture(t, md, present(), creds)
ctx := context.Background()
live := &backupsv1alpha1.Backup{}
if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil {
t.Fatal(err)
}
live.Finalizers = []string{backupFinalizer}
if err := f.c.Update(ctx, live); err != nil {
t.Fatal(err)
}
if err := f.c.Delete(ctx, live); err != nil {
t.Fatal(err)
}
f.reconcile(t)
f.wantFailed(t, "src-bk", "being deleted")
})
t.Run(name+", already gone", func(t *testing.T) {
f := newMariaDBRestoreFixture(t, md, present(), creds)
if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil {
t.Fatal(err)
}
f.reconcile(t)
f.wantFailed(t, "src-bk", "is gone")
})
}
t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) {
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds)
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return errors.New("apiserver unavailable")
}
return c.Get(ctx, key, obj, opts...)
},
})
f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable")
})
}
func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) {
creds := s3CredsSecret("tenant", "creds")
@@ -669,6 +669,20 @@ func (e mariadbRestoreRefused) Error() string { return string(e) }
// checked to still exist. A Backup that records no dump is resolved by
// legacyMariaDBRestoreTargetTime.
func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) {
// Deleting a Backup deletes its dump, and the operator restores the
// nearest other dump once its own is gone, so a restore must not start
// from a Backup being deleted. The copy in hand can predate the delete;
// the cleanup holds the Backup only for RestoreJobs it can already see.
live := &backupsv1alpha1.Backup{}
if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil {
if apierrors.IsNotFound(err) {
return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name))
}
return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err)
}
if !live.DeletionTimestamp.IsZero() {
return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name))
}
// A failed Job is Complete=True too, with reason JobFailed, and a
// BackupJob that took Complete=True for success made a Backup of it.
if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed {