From a27de054cabac5bdc15f7113bc772bf5963d3726 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Tue, 10 Nov 2026 00:07:37 +0300 Subject: [PATCH] fix(postgres): wait for the rollout another backup job's attach started Only the job that attaches the barman-cloud plugin waited for the instance rollout it causes. A second job of the same application that started meanwhile found the plugin already there, had no rollout of its own to wait for and created its cnpg.io/Backup at once, against instances CNPG was still recreating, and such a Backup fails for good. The queue does not hold it either, since the attaching job has no Backup yet. The attach record on the Cluster says when the plugin was attached, so a job without a rollout of its own now waits for instances newer than it while the attach is recent enough for that rollout to be running. An older attach has had its rollout judged by the job that made it. The record belongs to that job, so one this job cannot read, or one further ahead than controller clocks drift, is skipped rather than failing it; the latter would otherwise hold every later job of the application for as long as it lies ahead. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../cnpgstrategy_controller.go | 23 ++++++++++ .../cnpgstrategy_controller_test.go | 44 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index a9eb6fabc..bdc04bb0e 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -551,6 +551,29 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp }) apimeta.RemoveStatusCondition(&j.Status.Conditions, "Ready") rolledOut = true + } else if existingBackup == nil && attachment.attachedAt != "" { + // Another backup job attached the plugin. Its rollout can still be + // running while the attach is recent, and later than that the + // attaching job has failed it already. The record is that job's: + // one this job cannot read does not fail it, and neither does one + // further ahead than clocks drift, which every instance would + // predate for as long as it lies ahead. + recorded, _, err := cnpgRecordedAttach(attachment.attachedAt) + if err == nil && time.Until(recorded) > cnpgAttachClockSkewTolerance { + err = fmt.Errorf("%s is in the future", attachment.attachedAt) + } + if err != nil { + logger.Info("ignoring an unusable attach record another backup job left", + "cluster", clusterName, "annotation", cnpgPluginAttachedAtAnnotation, "error", err) + } else if time.Since(recorded) < cnpgDefaultBackupDeadline { + pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, metav1.NewTime(recorded)) + if err != nil { + return ctrl.Result{}, err + } + if pending != "" { + return r.waitCNPGBackupJob(ctx, j, "WaitingForPluginRollout", pending) + } + } } // The chart refuses a flavor change on a live Cluster, but the app keeps diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 85afa0c65..9c41d3931 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -3180,6 +3180,50 @@ func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) { } } +// A job that attached nothing itself still finds the plugin another backup +// job attached a moment ago, while CNPG rolls the instances for it; a Backup +// created then fails for good. It waits for that rollout as long as the +// attach is recent enough for it to be running, and an attach record it +// cannot read is not its own to fail on. +func TestReconcileCNPG_JobWithoutLatchWaitsForAnotherJobsRollout(t *testing.T) { + cases := []struct { + name string + record string + podCreated time.Time + wantBackup bool + }{ + {name: "recent attach, instance not rolled yet", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-10 * time.Minute), wantBackup: false}, + {name: "recent attach, instance rolled", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Minute), wantBackup: true}, + {name: "attach older than the rollout deadline", record: time.Now().Add(-2 * cnpgDefaultBackupDeadline).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-3 * cnpgDefaultBackupDeadline), wantBackup: true}, + {name: "unreadable record", record: "yesterday", podCreated: time.Now().Add(-time.Hour), wantBackup: true}, + {name: "record further ahead than clocks drift", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(tc.podCreated.Truncate(time.Second))) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed: %s", j.Status.Message) + } + if got := len(backups) == 1; got != tc.wantBackup { + t.Fatalf("cnpg.io/Backup created = %t, want %t; Ready = %+v", got, tc.wantBackup, apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if !tc.wantBackup { + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) + } + } + }) + } +} + // A plugin attached by someone else after the latch, another backup job or // the chart re-rendering a restored Cluster outside the platform flow, // started the rollout before this job's write, which only finds it there: