fix(postgres): read cnpg Backups from the apiserver before writing the destination

The queue that keeps two BackupJobs of one application from writing the
WAL archive destination at once listed cnpg.io/Backups from the
informer cache. The controller runs one reconcile at a time under
leader election, so a Backup created by one job's pass already exists
when the next pass starts, but the cache can still miss it. The next
job then took the queue as empty, rewrote the destination under the
running backup and left that backup's recorded destination naming a
place its WAL no longer went. A job could likewise miss its own fresh
Backup, write the destination again and create a second Backup.

The queue now lists Backups from the apiserver, and a job that finds
no Backup of its own in the cache checks the apiserver before it
writes or creates one. Only a leader handover, when two processes
reconcile for a moment, can still let two jobs through.

The pass that creates the Backup no longer looks it up a second time
right before the create. Nothing can create it in between, and once
the lookup can reach the apiserver, a failed second lookup would come
after the destination write and fail the job instead of retrying.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-10 00:08:41 +03:00
1 parent a6af14ce74
commit e5ccde82e2
2 files changed
+150 -15

No files matched your search

@@ -531,7 +531,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
"cluster", clusterName, "live", attachment.flavor, "application", specFlavor)
}
cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, cnpgDestinationOf(rendered))
cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, existingBackup, cnpgDestinationOf(rendered))
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure cnpg.io/Backup: %v", err))
}
@@ -876,18 +876,16 @@ func currentBarmanServerName(c *cnpgtypes.Cluster) string {
}
// ensureCNPGBackup creates a one-shot postgresql.cnpg.io/Backup CR labelled
// with the BackupJob, or returns the existing one if a previous reconcile
// already created it. Idempotency relies on the OwningJob labels.
// with the BackupJob, or returns existing, the job's Backup as looked up at
// the start of the pass. It does not look again: a reconcile of the same
// job cannot run in between, and a failed lookup here would come after the
// destination write and fail the job.
//
// written is the destination this pass wrote to the ObjectStore. A pass
// reaches the create only after that write: a job without a Backup writes
// the destination on every pass, and a pass that wrote nothing, or wrote it
// and waits for the plugin rollout, returns before this point.
func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string, written cnpgDestination) (*cnpgtypes.Backup, error) {
existing, err := r.findCNPGBackupForJob(ctx, j)
if err != nil {
return nil, err
}
func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string, existing *cnpgtypes.Backup, written cnpgDestination) (*cnpgtypes.Backup, error) {
if existing != nil {
return existing, nil
}
@@ -923,17 +921,27 @@ func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1
// findCNPGBackupForJob returns the cnpg.io/Backup labelled with the
// BackupJob's OwningJob{Name,Namespace}, if any. Returns (nil, nil) when
// no match is found.
//
// A miss in the cache is checked against the apiserver: the cache can lag
// behind a Backup the previous pass created, and taking that miss as no
// Backup would write the destination under it and create a second one.
func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backupsv1alpha1.BackupJob) (*cnpgtypes.Backup, error) {
list := &cnpgtypes.BackupList{}
if err := r.List(ctx, list,
opts := []client.ListOption{
client.InNamespace(j.Namespace),
client.MatchingLabels{
backupsv1alpha1.OwningJobNameLabel: j.Name,
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
},
); err != nil {
}
list := &cnpgtypes.BackupList{}
if err := r.List(ctx, list, opts...); err != nil {
return nil, err
}
if len(list.Items) == 0 {
if err := r.apiReader().List(ctx, list, opts...); err != nil {
return nil, err
}
}
if len(list.Items) == 0 {
return nil, nil
}
@@ -944,9 +952,12 @@ func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backu
// running and whose cnpg.io/Backup of clusterName has not finished, or "" when
// there is none. A Backup left behind by a finished or deleted job does not
// hold the queue.
//
// The Backups are read from the apiserver, not the cache, which can still
// miss a Backup another job's earlier pass created.
func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) {
list := &cnpgtypes.BackupList{}
if err := r.List(ctx, list, client.InNamespace(j.Namespace)); err != nil {
if err := r.apiReader().List(ctx, list, client.InNamespace(j.Namespace)); err != nil {
return "", err
}
var holders []string
@@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
@@ -2207,7 +2208,7 @@ func TestEnsureCNPGBackup_UsesPluginMethod(t *testing.T) {
c := newCNPGStrategyTestClient(t, j)
r := &BackupJobReconciler{Client: c}
bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", cnpgDestination{DestinationPath: "s3://bucket/"})
bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", nil, cnpgDestination{DestinationPath: "s3://bucket/"})
if err != nil {
t.Fatalf("ensureCNPGBackup: %v", err)
}
@@ -2567,8 +2568,8 @@ func (f *cnpgRolloutFixture) seedOtherBackupJob(name string, phase backupsv1alph
func TestReconcileCNPG_RunningBackupDoesNotReapplyObjectStore(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
// Two jobs can pass the queue together, since it reads the cache; once
// both have a Backup neither may wait on the other.
// Two jobs can still pass the queue together, across a leader handover;
// once both have a Backup neither may wait on the other.
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
writes := 0
@@ -5655,3 +5656,126 @@ func TestReconcileCNPG_ArtifactOfABackupWithoutARecord(t *testing.T) {
t.Fatalf("artifact = %+v, want one built from the strategy", artifact)
}
}
// cacheLaggingBehind makes the reconciler's cached client miss the named
// cnpg.io/Backups, as an informer does right after they are created, while
// its uncached reader sees them.
func (f *cnpgRolloutFixture) cacheLaggingBehind(hidden ...string) {
f.r.APIReader = f.c
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
if err := cl.List(ctx, list, opts...); err != nil {
return err
}
if backups, ok := list.(*cnpgtypes.BackupList); ok {
backups.Items = slices.DeleteFunc(backups.Items, func(b cnpgtypes.Backup) bool {
return slices.Contains(hidden, b.Name)
})
}
return nil
},
})
}
// Reconciles run one at a time, so a Backup created by one pass exists by
// the next; the queue has to see it even when the cache does not yet, or
// the next job rewrites the destination under it and the record on that
// Backup no longer names where its WAL goes.
func TestReconcileCNPG_QueueSeesABackupTheCacheHasNotCaught(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
f.cacheLaggingBehind("bj-other-cnpg")
j, backups := f.pass()
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" {
t.Fatalf("Ready = %+v, want WaitingForOtherBackup", ready)
}
if got := f.liveDestination(); got != "s3://other/" {
t.Fatalf("destination = %q, rewritten under the other job's Backup", got)
}
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups, want only the other job's", len(backups))
}
}
// The job's own Backup, missing from the cache right after it was created,
// is neither created again nor written under.
func TestReconcileCNPG_OwnBackupTheCacheHasNotCaught(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
f.cacheLaggingBehind("bj-cnpg")
_, backups := f.pass()
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups, want the job's one", len(backups))
}
if got := f.liveDestination(); got != "s3://other/" {
t.Fatalf("destination = %q, rewritten under the job's own Backup", got)
}
}
// failingBackupLists makes the uncached reader fail the cnpg.io/Backup lists
// for which fail, given whether the list selects the job's own Backup by
// label and how many such lists came before, returns true.
func (f *cnpgRolloutFixture) failingBackupLists(fail func(own bool, n int) bool) {
counts := map[bool]int{}
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
if _, ok := list.(*cnpgtypes.BackupList); ok {
o := &client.ListOptions{}
o.ApplyOptions(opts)
own := o.LabelSelector != nil && !o.LabelSelector.Empty()
counts[own]++
if fail(own, counts[own]) {
return apierrors.NewServiceUnavailable("apiserver unavailable")
}
}
return cl.List(ctx, list, opts...)
},
})
}
// A failed apiserver read of the Backups is retried, not taken as the job
// failing: nothing has been written yet that the retry would not redo.
func TestReconcileCNPG_BackupListErrorIsReturned(t *testing.T) {
for name, own := range map[string]bool{"own Backup": true, "queue": false} {
t.Run(name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.failingBackupLists(func(o bool, _ int) bool { return o == own })
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
t.Fatalf("reconcileCNPG swallowed the Backup list error")
}
got := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, got); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("BackupJob failed on a Backup list error: %s", got.Status.Message)
}
if dest := f.liveDestination(); dest != "s3://other/" {
t.Fatalf("destination = %q, written although the Backups could not be read", dest)
}
})
}
}
// The pass that writes the destination looks its own Backup up once, at the
// start: a second lookup before the create could only fail, after the write,
// and fail the job with it.
func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.failingBackupLists(func(own bool, n int) bool { return own && n > 1 })
j, backups := f.pass()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("BackupJob failed: %s", j.Status.Message)
}
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups))
}
}