fix(postgres): wait for the rollout another backup job's attach started

Only the job that attaches the barman-cloud plugin waited for the
instance rollout it causes. A second job of the same application that
started meanwhile found the plugin already there, had no rollout of its
own to wait for and created its cnpg.io/Backup at once, against
instances CNPG was still recreating, and such a Backup fails for good.
The queue does not hold it either, since the attaching job has no
Backup yet.

The attach record on the Cluster says when the plugin was attached, so
a job without a rollout of its own now waits for instances newer than
it while the attach is recent enough for that rollout to be running.
An older attach has had its rollout judged by the job that made it.
The record belongs to that job, so one this job cannot read, or one
further ahead than controller clocks drift, is skipped rather than
failing it; the latter would otherwise hold every later job of the
application for as long as it lies ahead.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-10 01:27:21 +03:00
1 parent d37701a327
commit a27de054ca
2 files changed
+67

No files matched your search

@@ -551,6 +551,29 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
})
apimeta.RemoveStatusCondition(&j.Status.Conditions, "Ready")
rolledOut = true
} else if existingBackup == nil && attachment.attachedAt != "" {
// Another backup job attached the plugin. Its rollout can still be
// running while the attach is recent, and later than that the
// attaching job has failed it already. The record is that job's:
// one this job cannot read does not fail it, and neither does one
// further ahead than clocks drift, which every instance would
// predate for as long as it lies ahead.
recorded, _, err := cnpgRecordedAttach(attachment.attachedAt)
if err == nil && time.Until(recorded) > cnpgAttachClockSkewTolerance {
err = fmt.Errorf("%s is in the future", attachment.attachedAt)
}
if err != nil {
logger.Info("ignoring an unusable attach record another backup job left",
"cluster", clusterName, "annotation", cnpgPluginAttachedAtAnnotation, "error", err)
} else if time.Since(recorded) < cnpgDefaultBackupDeadline {
pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, metav1.NewTime(recorded))
if err != nil {
return ctrl.Result{}, err
}
if pending != "" {
return r.waitCNPGBackupJob(ctx, j, "WaitingForPluginRollout", pending)
}
}
}
// The chart refuses a flavor change on a live Cluster, but the app keeps
@@ -3180,6 +3180,50 @@ func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) {
}
}
// A job that attached nothing itself still finds the plugin another backup
// job attached a moment ago, while CNPG rolls the instances for it; a Backup
// created then fails for good. It waits for that rollout as long as the
// attach is recent enough for it to be running, and an attach record it
// cannot read is not its own to fail on.
func TestReconcileCNPG_JobWithoutLatchWaitsForAnotherJobsRollout(t *testing.T) {
cases := []struct {
name string
record string
podCreated time.Time
wantBackup bool
}{
{name: "recent attach, instance not rolled yet", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-10 * time.Minute), wantBackup: false},
{name: "recent attach, instance rolled", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Minute), wantBackup: true},
{name: "attach older than the rollout deadline", record: time.Now().Add(-2 * cnpgDefaultBackupDeadline).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-3 * cnpgDefaultBackupDeadline), wantBackup: true},
{name: "unreadable record", record: "yesterday", podCreated: time.Now().Add(-time.Hour), wantBackup: true},
{name: "record further ahead than clocks drift", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))
cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record}
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster,
newRolloutInstancePod(tc.podCreated.Truncate(time.Second)))
j, backups, res := f.passResult()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("the job failed: %s", j.Status.Message)
}
if got := len(backups) == 1; got != tc.wantBackup {
t.Fatalf("cnpg.io/Backup created = %t, want %t; Ready = %+v", got, tc.wantBackup, apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
if !tc.wantBackup {
if res.RequeueAfter == 0 {
t.Errorf("a waiting job is not requeued: %+v", res)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" {
t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready)
}
}
})
}
}
// A plugin attached by someone else after the latch, another backup job or
// the chart re-rendering a restored Cluster outside the platform flow,
// started the rollout before this job's write, which only finds it there: