mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): retry a failed lookup of a MariaDB operator Restore
The lookup now goes to the API server, which can time out or throttle, and a failed lookup failed the RestoreJob. It says nothing about whether the operator Restore exists, so the RestoreJob could be failed while its Restore was running, and the Backup's cleanup would then stop waiting for it and delete the dump the Restore reads. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
14a842ad16
commit
ee6404a0c8
2 files changed
+25
-1
No files matched your search
@@ -260,6 +260,28 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) {
|
||||
t.Errorf("created %d operator Restores; want the existing one followed", len(got))
|
||||
}
|
||||
})
|
||||
// A failed lookup says nothing about whether the operator Restore
|
||||
// exists, so the RestoreJob must not be failed under one that runs.
|
||||
t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds)
|
||||
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
|
||||
List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
|
||||
if _, ok := list.(*mariadbtypes.RestoreList); ok {
|
||||
return errors.New("etcdserver: request timed out")
|
||||
}
|
||||
return c.List(ctx, list, opts...)
|
||||
},
|
||||
})
|
||||
if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil {
|
||||
t.Error("err = nil, want the failed lookup returned for a retry")
|
||||
}
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message)
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores without knowing whether one exists", len(got))
|
||||
}
|
||||
})
|
||||
t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds)
|
||||
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
|
||||
|
||||
@@ -598,7 +598,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
|
||||
mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob)
|
||||
if err != nil {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err))
|
||||
// Retried: failing the RestoreJob here could fail it under an
|
||||
// operator Restore that is running.
|
||||
return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err)
|
||||
}
|
||||
// The dump is chosen once, as the operator Restore is created; one that
|
||||
// already exists has made its choice.
|
||||
|
||||
Reference in new issue
Block a user