fix(postgres): read restores from the apiserver before writing the destination

A backup job that is about to write the WAL archive destination first
checks for a restore of its application, because a write landing during
the restore puts the restored Cluster back on the source's WAL prefix
and the restore fails with an expected-empty-archive error after it has
purged the target. That check read RestoreJobs from the informer cache,
which can still miss one created a moment ago.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-10 00:08:41 +03:00
1 parent 8a631311a7
commit b4551eb379
3 files changed
+33 -4

No files matched your search

@@ -38,7 +38,7 @@ const ConditionReasonRestoreInProgress = "RestoreInProgress"
// restored cluster back on the source's WAL prefix, where its recovery fails
// with "Expected empty archive". Plan-created runs are the ones that hit
// this, since nothing else looks at restores before creating a BackupJob.
func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
func activeRestoreTargeting(ctx context.Context, c client.Reader, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
app = NormalizeApplicationRef(app)
list := &backupsv1alpha1.RestoreJobList{}
if err := c.List(ctx, list, client.InNamespace(namespace)); err != nil {
@@ -80,7 +80,7 @@ func activeRestoreTargeting(ctx context.Context, c client.Client, namespace stri
// reads options.targetNamespace, so the option is looked at only when the
// Backup's strategy is Velero — the CNPG and Job-based drivers ignore it and
// restore in place.
func restoreJobTarget(ctx context.Context, c client.Client, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) {
func restoreJobTarget(ctx context.Context, c client.Reader, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) {
backup := &backupsv1alpha1.Backup{}
if err := c.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: rj.Spec.BackupRef.Name}, backup); err != nil {
if client.IgnoreNotFound(err) == nil {
@@ -365,8 +365,9 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
if writeDestination {
// The gate in Reconcile sees only a job that has not started, and
// this one may have waited in the queue for as long as a restore
// takes to begin.
restore, err := activeRestoreTargeting(ctx, r.Client, j.Namespace, j.Spec.ApplicationRef)
// takes to begin. Read from the apiserver: the cache can still miss
// a RestoreJob created a moment ago.
restore, err := activeRestoreTargeting(ctx, r.apiReader(), j.Namespace, j.Spec.ApplicationRef)
if err != nil {
return ctrl.Result{}, err
}
@@ -3209,6 +3209,34 @@ func (f *cnpgRolloutFixture) seedRestoreOfApp() *backupsv1alpha1.RestoreJob {
return restore
}
// The restore check before the destination write reads the apiserver: the
// cache can still miss a RestoreJob created a moment ago, and a write landing
// in that gap can put the restored Cluster back on the source's WAL prefix.
func TestReconcileCNPG_RestoreCheckReadsPastTheCache(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
f.seedRestoreOfApp()
f.r.APIReader = f.c
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok {
return nil
}
return cl.List(ctx, list, opts...)
},
})
j, backups := f.pass()
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination written during a restore the cache had not caught: %q", got)
}
if len(backups) != 0 {
t.Errorf("a cnpg.io/Backup was created during a restore the cache had not caught: %+v", backups)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress {
t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress)
}
}
// A job past its start writes the WAL archive destination only while no
// restore rewrites its application. The restore gate in Reconcile looks only
// at a job that has not started, and a job released from the queue, or one