From e5ccde82e2258a55e97b0dd6e5375caed5b59b86 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 10:47:21 +0300 Subject: [PATCH] fix(postgres): read cnpg Backups from the apiserver before writing the destination The queue that keeps two BackupJobs of one application from writing the WAL archive destination at once listed cnpg.io/Backups from the informer cache. The controller runs one reconcile at a time under leader election, so a Backup created by one job's pass already exists when the next pass starts, but the cache can still miss it. The next job then took the queue as empty, rewrote the destination under the running backup and left that backup's recorded destination naming a place its WAL no longer went. A job could likewise miss its own fresh Backup, write the destination again and create a second Backup. The queue now lists Backups from the apiserver, and a job that finds no Backup of its own in the cache checks the apiserver before it writes or creates one. Only a leader handover, when two processes reconcile for a moment, can still let two jobs through. The pass that creates the Backup no longer looks it up a second time right before the create. Nothing can create it in between, and once the lookup can reach the apiserver, a failed second lookup would come after the destination write and fail the job instead of retrying. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../cnpgstrategy_controller.go | 35 +++-- .../cnpgstrategy_controller_test.go | 130 +++++++++++++++++- 2 files changed, 150 insertions(+), 15 deletions(-) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 190e2db6c..3eb3421d6 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -531,7 +531,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp "cluster", clusterName, "live", attachment.flavor, "application", specFlavor) } - cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, cnpgDestinationOf(rendered)) + cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, existingBackup, cnpgDestinationOf(rendered)) if err != nil { return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure cnpg.io/Backup: %v", err)) } @@ -876,18 +876,16 @@ func currentBarmanServerName(c *cnpgtypes.Cluster) string { } // ensureCNPGBackup creates a one-shot postgresql.cnpg.io/Backup CR labelled -// with the BackupJob, or returns the existing one if a previous reconcile -// already created it. Idempotency relies on the OwningJob labels. +// with the BackupJob, or returns existing, the job's Backup as looked up at +// the start of the pass. It does not look again: a reconcile of the same +// job cannot run in between, and a failed lookup here would come after the +// destination write and fail the job. // // written is the destination this pass wrote to the ObjectStore. A pass // reaches the create only after that write: a job without a Backup writes // the destination on every pass, and a pass that wrote nothing, or wrote it // and waits for the plugin rollout, returns before this point. -func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string, written cnpgDestination) (*cnpgtypes.Backup, error) { - existing, err := r.findCNPGBackupForJob(ctx, j) - if err != nil { - return nil, err - } +func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string, existing *cnpgtypes.Backup, written cnpgDestination) (*cnpgtypes.Backup, error) { if existing != nil { return existing, nil } @@ -923,17 +921,27 @@ func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1 // findCNPGBackupForJob returns the cnpg.io/Backup labelled with the // BackupJob's OwningJob{Name,Namespace}, if any. Returns (nil, nil) when // no match is found. +// +// A miss in the cache is checked against the apiserver: the cache can lag +// behind a Backup the previous pass created, and taking that miss as no +// Backup would write the destination under it and create a second one. func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backupsv1alpha1.BackupJob) (*cnpgtypes.Backup, error) { - list := &cnpgtypes.BackupList{} - if err := r.List(ctx, list, + opts := []client.ListOption{ client.InNamespace(j.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: j.Name, backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, }, - ); err != nil { + } + list := &cnpgtypes.BackupList{} + if err := r.List(ctx, list, opts...); err != nil { return nil, err } + if len(list.Items) == 0 { + if err := r.apiReader().List(ctx, list, opts...); err != nil { + return nil, err + } + } if len(list.Items) == 0 { return nil, nil } @@ -944,9 +952,12 @@ func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backu // running and whose cnpg.io/Backup of clusterName has not finished, or "" when // there is none. A Backup left behind by a finished or deleted job does not // hold the queue. +// +// The Backups are read from the apiserver, not the cache, which can still +// miss a Backup another job's earlier pass created. func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { list := &cnpgtypes.BackupList{} - if err := r.List(ctx, list, client.InNamespace(j.Namespace)); err != nil { + if err := r.apiReader().List(ctx, list, client.InNamespace(j.Namespace)); err != nil { return "", err } var holders []string diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 23e4c182f..de525f0a7 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "testing" "time" @@ -2207,7 +2208,7 @@ func TestEnsureCNPGBackup_UsesPluginMethod(t *testing.T) { c := newCNPGStrategyTestClient(t, j) r := &BackupJobReconciler{Client: c} - bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", cnpgDestination{DestinationPath: "s3://bucket/"}) + bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", nil, cnpgDestination{DestinationPath: "s3://bucket/"}) if err != nil { t.Fatalf("ensureCNPGBackup: %v", err) } @@ -2567,8 +2568,8 @@ func (f *cnpgRolloutFixture) seedOtherBackupJob(name string, phase backupsv1alph func TestReconcileCNPG_RunningBackupDoesNotReapplyObjectStore(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") - // Two jobs can pass the queue together, since it reads the cache; once - // both have a Backup neither may wait on the other. + // Two jobs can still pass the queue together, across a leader handover; + // once both have a Backup neither may wait on the other. f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") writes := 0 @@ -5655,3 +5656,126 @@ func TestReconcileCNPG_ArtifactOfABackupWithoutARecord(t *testing.T) { t.Fatalf("artifact = %+v, want one built from the strategy", artifact) } } + +// cacheLaggingBehind makes the reconciler's cached client miss the named +// cnpg.io/Backups, as an informer does right after they are created, while +// its uncached reader sees them. +func (f *cnpgRolloutFixture) cacheLaggingBehind(hidden ...string) { + f.r.APIReader = f.c + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if backups, ok := list.(*cnpgtypes.BackupList); ok { + backups.Items = slices.DeleteFunc(backups.Items, func(b cnpgtypes.Backup) bool { + return slices.Contains(hidden, b.Name) + }) + } + return nil + }, + }) +} + +// Reconciles run one at a time, so a Backup created by one pass exists by +// the next; the queue has to see it even when the cache does not yet, or +// the next job rewrites the destination under it and the record on that +// Backup no longer names where its WAL goes. +func TestReconcileCNPG_QueueSeesABackupTheCacheHasNotCaught(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + f.cacheLaggingBehind("bj-other-cnpg") + + j, backups := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" { + t.Fatalf("Ready = %+v, want WaitingForOtherBackup", ready) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("destination = %q, rewritten under the other job's Backup", got) + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want only the other job's", len(backups)) + } +} + +// The job's own Backup, missing from the cache right after it was created, +// is neither created again nor written under. +func TestReconcileCNPG_OwnBackupTheCacheHasNotCaught(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.cacheLaggingBehind("bj-cnpg") + + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want the job's one", len(backups)) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("destination = %q, rewritten under the job's own Backup", got) + } +} + +// failingBackupLists makes the uncached reader fail the cnpg.io/Backup lists +// for which fail, given whether the list selects the job's own Backup by +// label and how many such lists came before, returns true. +func (f *cnpgRolloutFixture) failingBackupLists(fail func(own bool, n int) bool) { + counts := map[bool]int{} + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*cnpgtypes.BackupList); ok { + o := &client.ListOptions{} + o.ApplyOptions(opts) + own := o.LabelSelector != nil && !o.LabelSelector.Empty() + counts[own]++ + if fail(own, counts[own]) { + return apierrors.NewServiceUnavailable("apiserver unavailable") + } + } + return cl.List(ctx, list, opts...) + }, + }) +} + +// A failed apiserver read of the Backups is retried, not taken as the job +// failing: nothing has been written yet that the retry would not redo. +func TestReconcileCNPG_BackupListErrorIsReturned(t *testing.T) { + for name, own := range map[string]bool{"own Backup": true, "queue": false} { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.failingBackupLists(func(o bool, _ int) bool { return o == own }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the Backup list error") + } + got := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, got); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed on a Backup list error: %s", got.Status.Message) + } + if dest := f.liveDestination(); dest != "s3://other/" { + t.Fatalf("destination = %q, written although the Backups could not be read", dest) + } + }) + } +} + +// The pass that writes the destination looks its own Backup up once, at the +// start: a second lookup before the create could only fail, after the write, +// and fail the job with it. +func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.failingBackupLists(func(own bool, n int) bool { return own && n > 1 }) + + j, backups := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups)) + } +}