mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(postgres): read restores from the apiserver before writing the destination
A backup job that is about to write the WAL archive destination first checks for a restore of its application, because a write landing during the restore puts the restored Cluster back on the source's WAL prefix and the restore fails with an expected-empty-archive error after it has purged the target. That check read RestoreJobs from the informer cache, which can still miss one created a moment ago. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
8a631311a7
commit
b4551eb379
3 files changed
+33
-4
No files matched your search
@@ -38,7 +38,7 @@ const ConditionReasonRestoreInProgress = "RestoreInProgress"
|
||||
// restored cluster back on the source's WAL prefix, where its recovery fails
|
||||
// with "Expected empty archive". Plan-created runs are the ones that hit
|
||||
// this, since nothing else looks at restores before creating a BackupJob.
|
||||
func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
|
||||
func activeRestoreTargeting(ctx context.Context, c client.Reader, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
|
||||
app = NormalizeApplicationRef(app)
|
||||
list := &backupsv1alpha1.RestoreJobList{}
|
||||
if err := c.List(ctx, list, client.InNamespace(namespace)); err != nil {
|
||||
@@ -80,7 +80,7 @@ func activeRestoreTargeting(ctx context.Context, c client.Client, namespace stri
|
||||
// reads options.targetNamespace, so the option is looked at only when the
|
||||
// Backup's strategy is Velero — the CNPG and Job-based drivers ignore it and
|
||||
// restore in place.
|
||||
func restoreJobTarget(ctx context.Context, c client.Client, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) {
|
||||
func restoreJobTarget(ctx context.Context, c client.Reader, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) {
|
||||
backup := &backupsv1alpha1.Backup{}
|
||||
if err := c.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: rj.Spec.BackupRef.Name}, backup); err != nil {
|
||||
if client.IgnoreNotFound(err) == nil {
|
||||
|
||||
@@ -365,8 +365,9 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
if writeDestination {
|
||||
// The gate in Reconcile sees only a job that has not started, and
|
||||
// this one may have waited in the queue for as long as a restore
|
||||
// takes to begin.
|
||||
restore, err := activeRestoreTargeting(ctx, r.Client, j.Namespace, j.Spec.ApplicationRef)
|
||||
// takes to begin. Read from the apiserver: the cache can still miss
|
||||
// a RestoreJob created a moment ago.
|
||||
restore, err := activeRestoreTargeting(ctx, r.apiReader(), j.Namespace, j.Spec.ApplicationRef)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
@@ -3209,6 +3209,34 @@ func (f *cnpgRolloutFixture) seedRestoreOfApp() *backupsv1alpha1.RestoreJob {
|
||||
return restore
|
||||
}
|
||||
|
||||
// The restore check before the destination write reads the apiserver: the
|
||||
// cache can still miss a RestoreJob created a moment ago, and a write landing
|
||||
// in that gap can put the restored Cluster back on the source's WAL prefix.
|
||||
func TestReconcileCNPG_RestoreCheckReadsPastTheCache(t *testing.T) {
|
||||
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
|
||||
f.seedRestoreOfApp()
|
||||
f.r.APIReader = f.c
|
||||
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
|
||||
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
|
||||
if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok {
|
||||
return nil
|
||||
}
|
||||
return cl.List(ctx, list, opts...)
|
||||
},
|
||||
})
|
||||
|
||||
j, backups := f.pass()
|
||||
if got := f.liveDestination(); got != "s3://other/" {
|
||||
t.Errorf("destination written during a restore the cache had not caught: %q", got)
|
||||
}
|
||||
if len(backups) != 0 {
|
||||
t.Errorf("a cnpg.io/Backup was created during a restore the cache had not caught: %+v", backups)
|
||||
}
|
||||
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress {
|
||||
t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress)
|
||||
}
|
||||
}
|
||||
|
||||
// A job past its start writes the WAL archive destination only while no
|
||||
// restore rewrites its application. The restore gate in Reconcile looks only
|
||||
// at a job that has not started, and a job released from the queue, or one
|
||||
|
||||
Reference in new issue
Block a user