fix(postgres): time the plugin rollout from the write that attached it

The PluginRolledOut latch is recorded before the Cluster write that
attaches the barman-cloud plugin, and its time was used both as the
start of the rollout deadline and as the age every instance pod must
exceed to count as rolled. Now that a failing write is retried for up
to the backup deadline, the attach can come long after the latch: the
deadline was then spent before the rollout began, and a pod recreated
between the latch and the attach, without the plugin sidecar, passed
as rolled, so a Backup was created that CloudNativePG fails for good.

The latch now carries its own reason, and the write that finds the
plugin missing and attaches it re-stamps the condition, once, with the
time it was attempted. A plugin someone else attached after the latch,
another backup job or a restore, keeps the latch time: the rollout it
started can be over by the time this job writes, and its instances
would otherwise never count as rolled.

Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
Assisted-by: LLM
This commit is contained in:
Alexey Artamonov committed 2026-11-09 23:52:25 +03:00
1 parent 570fcec725
commit 567f07d708
2 files changed
+140

No files matched your search

@@ -197,6 +197,11 @@ const (
// instance pods must postdate (see cnpgPluginRolloutPending).
backupCondPluginRolledOut = "PluginRolledOut"
// Reason of the PluginRolledOut latch recorded before the write that
// attaches the plugin. The write that attaches it replaces the latch with
// its own time, which retries can put long after the latch.
cnpgReasonAttachingPlugin = "AttachingPlugin"
// Condition Type recorded while writing the WAL archive destination keeps
// failing. Its LastTransitionTime marks the first failure, so the retry
// budget is not spent by time the job waited for another backup of the
@@ -356,6 +361,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
}
attemptedAt := metav1.Now()
attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination)
if err != nil {
if apierrors.IsNotFound(err) {
@@ -419,9 +425,34 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: backupCondPluginRolledOut,
Status: metav1.ConditionFalse,
Reason: cnpgReasonAttachingPlugin,
Message: message,
})
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "WaitingForPluginRollout",
Message: message,
})
if err := r.Status().Update(ctx, j); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
}
if rollout.Reason == cnpgReasonAttachingPlugin && attachment.missing {
// Instances recreated between the latch and this write run
// without the plugin, and the rollout deadline starts here. A
// plugin someone else attached since the latch keeps it: the
// rollout that attach started may already be over.
message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName)
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondPluginRolledOut)
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: backupCondPluginRolledOut,
Status: metav1.ConditionFalse,
Reason: "WaitingForPluginRollout",
Message: message,
LastTransitionTime: attemptedAt,
})
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
@@ -2814,6 +2814,115 @@ func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) {
}
}
// The rollout is measured from the write that attached the plugin, not from
// the latch recorded before it: retries of a failing write can lie between
// the two, and an instance recreated in that time runs without the plugin.
func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) {
cases := []struct {
name string
// podBetween recreates the instance after the latch and before the
// attach; otherwise it predates both.
podBetween bool
}{
{name: "an instance recreated before the attach still has to roll", podBetween: true},
{name: "retries before the attach do not spend the rollout deadline"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second)
oldPod := newRolloutInstancePod(expired.Add(-time.Hour))
f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase), oldPod)
j, _ := f.pass() // latch
latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
if latch == nil {
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
}
latch.LastTransitionTime = metav1.NewTime(expired)
if err := f.c.Status().Update(context.Background(), j); err != nil {
t.Fatalf("age the latch: %v", err)
}
failing := true
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if _, ok := obj.(*cnpgtypes.Cluster); ok && failing {
return apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io"`))
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("a failed attach failed the job: %s", j.Status.Message)
}
if tc.podBetween {
f.replacePod(oldPod, newRolloutInstancePod(expired.Add(time.Minute)))
}
failing = false
attachAt := time.Now().Truncate(time.Second)
j, backups := f.pass()
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created in the pass that attached the plugin: %+v", backups)
}
rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) {
t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt)
}
attached := &cnpgtypes.Cluster{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil {
t.Fatalf("get Cluster: %v", err)
}
f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...))
j, backups = f.pass()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("a rollout that began with the attach failed: %s", j.Status.Message)
}
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || !strings.Contains(ready.Message, "predates") {
t.Errorf("Ready = %+v, want the instance that predates the plugin", ready)
}
})
}
}
// A plugin attached by someone else after the latch, another backup job or
// a restore, started the rollout before this job's write, which only finds
// it there: instances that rolled since the latch carry the plugin.
func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) {
oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second))
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase), oldPod)
j, _ := f.pass() // latch
latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
if latch == nil {
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
}
latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second)
latch.LastTransitionTime = metav1.NewTime(latchedAt)
if err := f.c.Status().Update(context.Background(), j); err != nil {
t.Fatalf("age the latch: %v", err)
}
f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)))
f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(5*time.Minute)))
var backups []cnpgtypes.Backup
for range 3 {
if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
break
}
}
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups against instances that rolled with the plugin; Ready = %+v",
len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
}
// Waiting for another backup resets the retry budget of the destination
// write, so the queue does not spend it.
func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) {