mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(mariadb): pin legacy Backups to a proven dump, refuse failed runs
A Backup taken before Backups recorded their dump gives a restore nothing to pin. Refusing such Backups would leave every existing one unrestorable, so a restore of one now makes the same ownership-proven match the backup side makes, against the operator Backup it names, and restores that dump by its timestamp. When no dump can be proven to be the run's own, or the storage cannot be read, mariadb-operator restores the newest dump under the prefix as it always did, and a BestEffortDump Warning on the RestoreJob says so and why. An operator Backup whose Job failed is Complete=True with reason JobFailed, and a BackupJob that took Complete=True for success made a Backup of it. That run left no dump of its own, and restoring it would quietly bring back another run's dump, so the RestoreJob now fails instead. The condition is read by its reason, because its status does not tell a failure from a success. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
2 files changed
+156
-14
No files matched your search
@@ -274,13 +274,128 @@ func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) {
|
||||
l := &fakeS3Lister{err: errors.New("Access Denied.")}
|
||||
f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds"))
|
||||
f.reconcile(t)
|
||||
f.wantRestore(t, "")
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls)
|
||||
// TestReconcileMariaDBRestore_LegacyBackup drives Backups that record no
|
||||
// dump: one proven to belong to the source run is restored by its
|
||||
// timestamp, and anything else restores the newest dump with a Warning.
|
||||
func TestReconcileMariaDBRestore_LegacyBackup(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
own := s3ObjectInfo{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}
|
||||
newer := s3ObjectInfo{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}
|
||||
second := s3ObjectInfo{Key: mdbRestorePrefix + "backup.2026-10-21T10:00:30Z.gzip.sql", LastModified: mdbRestoreT0.Add(35 * time.Second)}
|
||||
|
||||
t.Run("a dump proven to be the run's own is restored by its timestamp", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: []s3ObjectInfo{own, newer}}, creds)
|
||||
f.reconcile(t)
|
||||
f.wantRestore(t, mdbRestoreTime)
|
||||
if ev := drainEvents(f.events); len(ev) != 0 {
|
||||
t.Errorf("events %q; want none for a proven dump", ev)
|
||||
}
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
lister *fakeS3Lister
|
||||
cause string
|
||||
}{
|
||||
{name: "two dumps in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second, newer}}, cause: "cannot be told apart"},
|
||||
{name: "no dump in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{newer}}, cause: "no dump in the bucket belongs"},
|
||||
{name: "a storage that cannot be read", lister: &fakeS3Lister{err: errors.New("Access Denied.")}, cause: "Access Denied."},
|
||||
}
|
||||
wantBestEffort := func(t *testing.T, f *mariadbRestoreFixture, cause string) {
|
||||
t.Helper()
|
||||
f.wantRestore(t, "")
|
||||
ev := drainEvents(f.events)
|
||||
if len(ev) != 1 || !strings.HasPrefix(ev[0], "Warning BestEffortDump ") || !strings.Contains(ev[0], cause) || !strings.Contains(ev[0], "src-bk records no dump") {
|
||||
t.Errorf("events %q; want one BestEffortDump warning naming the Backup and %q", ev, cause)
|
||||
}
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name+" restores the newest dump with a warning", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, nil, tc.lister, creds)
|
||||
f.reconcile(t)
|
||||
wantBestEffort(t, f, tc.cause)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{}, creds)
|
||||
f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
var err error
|
||||
withShortDumpCheckTimeout(t, func() { _, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) })
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileMariaDBRestore: %v", err)
|
||||
}
|
||||
wantBestEffort(t, f, context.DeadlineExceeded.Error())
|
||||
})
|
||||
}
|
||||
|
||||
// TestReconcileMariaDBRestore_FailedBackupRun drives a Backup made of an
|
||||
// operator Backup whose Job failed: there is no dump of its own to
|
||||
// restore, whatever the Backup records.
|
||||
func TestReconcileMariaDBRestore_FailedBackupRun(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
failed := func(status metav1.ConditionStatus) []metav1.Condition {
|
||||
return []metav1.Condition{{
|
||||
Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: mariadbtypes.ConditionReasonJobFailed,
|
||||
Message: "Failed", LastTransitionTime: metav1.NewTime(mdbRestoreT0.Add(3 * time.Minute)),
|
||||
}}
|
||||
}
|
||||
listing := []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
driverMD map[string]string
|
||||
status metav1.ConditionStatus
|
||||
}{
|
||||
{name: "Complete=True/JobFailed with no recorded dump", status: metav1.ConditionTrue},
|
||||
{name: "Complete=True/JobFailed with a recorded dump", driverMD: recordedDump(mdbRestoreDump), status: metav1.ConditionTrue},
|
||||
{name: "Complete=False/JobFailed", status: metav1.ConditionFalse},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name+" fails the RestoreJob", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: listing}
|
||||
f := newMariaDBRestoreFixture(t, tc.driverMD, l, creds)
|
||||
f.src.Status.Conditions = failed(tc.status)
|
||||
if err := f.c.Update(context.Background(), f.src); err != nil {
|
||||
t.Fatalf("update operator Backup: %v", err)
|
||||
}
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, "k8s.mariadb.com/Backup tenant/op-bk failed", "left no dump to restore")
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a failed backup run", l.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("an operator Restore created earlier is followed", func(t *testing.T) {
|
||||
existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "rj-abcde",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: "rj",
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
||||
},
|
||||
}}
|
||||
f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: listing}, creds, existing)
|
||||
f.src.Status.Conditions = failed(metav1.ConditionTrue)
|
||||
if err := f.c.Update(context.Background(), f.src); err != nil {
|
||||
t.Fatalf("update operator Backup: %v", err)
|
||||
}
|
||||
f.reconcile(t)
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Errorf("RestoreJob failed although its operator Restore was already created: %s", f.rj.Status.Message)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func drainEvents(r *record.FakeRecorder) []string {
|
||||
var out []string
|
||||
for {
|
||||
select {
|
||||
case e := <-r.Events:
|
||||
out = append(out, e)
|
||||
default:
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -603,7 +603,7 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
// The dump is chosen once, as the operator Restore is created; one that
|
||||
// already exists has made its choice.
|
||||
if mdbRestore == nil {
|
||||
targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup)
|
||||
targetTime, err := r.mariadbRestoreTargetTime(ctx, restoreJob, backup, srcBackup)
|
||||
var refused mariadbRestoreRefused
|
||||
if errors.As(err, &refused) {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, refused.Error())
|
||||
@@ -666,19 +666,26 @@ func (e mariadbRestoreRefused) Error() string { return string(e) }
|
||||
// Restore for backup: the timestamp in the name of the dump backup
|
||||
// records. mariadb-operator restores the dump whose timestamp equals the
|
||||
// target and, when there is none, the nearest one, so the dump is first
|
||||
// checked to still exist. Empty when backup records no dump, which leaves
|
||||
// the operator to take the newest dump under the prefix.
|
||||
func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) {
|
||||
// checked to still exist. A Backup that records no dump is resolved by
|
||||
// legacyMariaDBRestoreTargetTime.
|
||||
func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) {
|
||||
// A failed Job is Complete=True too, with reason JobFailed, and a
|
||||
// BackupJob that took Complete=True for success made a Backup of it.
|
||||
if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf(
|
||||
"the backup run of k8s.mariadb.com/Backup %s/%s failed (Complete=%s/%s) and left no dump to restore; restoring would bring back another run's dump",
|
||||
src.Namespace, src.Name, cond.Status, cond.Reason))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout)
|
||||
defer cancel()
|
||||
name := backup.Spec.DriverMetadata[mariadbDumpFileKey]
|
||||
if name == "" {
|
||||
return "", nil
|
||||
return r.legacyMariaDBRestoreTargetTime(ctx, rj, backup, src), nil
|
||||
}
|
||||
ts, ok := parseMariaDBLogicalBackupFile(name)
|
||||
if !ok {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout)
|
||||
defer cancel()
|
||||
exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err)
|
||||
@@ -692,6 +699,26 @@ func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, bac
|
||||
return ts.UTC().Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
// legacyMariaDBRestoreTargetTime chooses the dump of a Backup taken before
|
||||
// Backups recorded theirs: the one findMariaDBDump proves belongs to src,
|
||||
// with the completion time the size probe was given, which takenAt keeps.
|
||||
// When none is proven, for any reason, it returns empty, and the operator
|
||||
// restores the newest dump under the prefix as such a restore always did:
|
||||
// refusing would leave these Backups unrestorable, and the controller may
|
||||
// never have been able to reach their storage. A Warning event says the
|
||||
// dump is a best effort.
|
||||
func (r *RestoreJobReconciler) legacyMariaDBRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) string {
|
||||
dump, err := findMariaDBDump(ctx, r.objectStore(), src, backup.Spec.TakenAt.Time)
|
||||
if err != nil {
|
||||
getLogger(ctx).Info("no dump proven for a MariaDB Backup that records none; restoring the newest", "restorejob", rj.Name, "backup", backup.Name, "error", err)
|
||||
r.Recorder.Eventf(rj, corev1.EventTypeWarning, "BestEffortDump",
|
||||
"Backup %s records no dump and none in storage could be proven to belong to k8s.mariadb.com/Backup %s/%s (%v); mariadb-operator restores the newest dump under its prefix as a best effort, which may be another backup run's",
|
||||
backup.Name, src.Namespace, src.Name, err)
|
||||
return ""
|
||||
}
|
||||
return dump.Timestamp.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False
|
||||
// saying why, while the dump it asks for cannot be checked, and fails it
|
||||
// with the cause once the restore deadline has passed.
|
||||
|
||||
Reference in new issue
Block a user