fix(postgres): restart the plugin rollout whenever a backup job attaches it

Only the first attach after the latch restarted the rollout. In the
platform flow the chart renders no spec.plugins, so an in-place restore
recreates the Cluster without the plugin, and a backup job held through
that restore attaches it again once released. The rollout condition
then still carried its earlier start, the restored instances were newer
than it and ready, and the Backup was created in the same pass, before
CloudNativePG had rolled anything; such a Backup fails with "requested
plugin is not available" or "instance manager was restarted".

Every write of the job that finds the plugin missing now starts the
rollout over. A plugin attached by another writer is still timed from
the latch.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-09 23:52:25 +03:00
1 parent 32ca678077
commit 258c5359b6
2 files changed
+57 -13

No files matched your search

@@ -175,12 +175,13 @@ const (
// against an unreachable target reproduces the same FATAL.
cnpgRecoveryMaxInspectPods = 5
// Cap on the wall-clock time a BackupJob waits for the barman-cloud
// plugin rollout, counted from the PluginRolledOut condition or from the
// release recorded in DestinationWriteHeld, whichever is later, or retries
// writing the WAL archive destination, counted from the first failure
// since the last successful write or the last wait for another backup or
// a restore, before the cnpg.io/Backup exists. A Backup that exists is
// Cap on the wall-clock time a BackupJob spends before its
// cnpg.io/Backup exists on each of two waits. The plugin rollout is
// counted from the PluginRolledOut condition, or from the release
// recorded in DestinationWriteHeld when that is later. Retrying the WAL
// archive destination write is counted from the first failure since the
// last successful write or the last wait for another backup or a
// restore. A Backup that exists is
// not capped: CloudNativePG keeps a healthy one pending while another
// backup of the same Cluster runs, or while it restarts one on another
// instance, so no fixed deadline tells those apart from a stuck one.
@@ -200,8 +201,9 @@ const (
backupCondPluginRolledOut = "PluginRolledOut"
// Reason of the PluginRolledOut latch recorded before the write that
// attaches the plugin. The write that attaches it replaces the latch with
// its own time, which retries can put long after the latch.
// attaches the plugin. Every write of this job that attaches it replaces
// the condition with its own time, which retries can put long after the
// latch.
cnpgReasonAttachingPlugin = "AttachingPlugin"
// Condition Type recorded while writing the WAL archive destination keeps
@@ -454,9 +456,10 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
}
if rollout.Reason == cnpgReasonAttachingPlugin && attachment.missing {
// Instances recreated between the latch and this write run
// without the plugin, and the rollout deadline starts here. A
if attachment.missing {
// This write attached the plugin, so its rollout starts here:
// instances recreated since the latch, or by a restore that
// re-rendered the Cluster without the plugin, run without it. A
// plugin someone else attached since the latch keeps it: the
// rollout that attach started may already be over.
message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName)
@@ -2830,6 +2830,11 @@ func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) {
if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" {
t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond)
}
attached := &cnpgtypes.Cluster{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil {
t.Fatalf("get Cluster: %v", err)
}
f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...))
rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
rollout.LastTransitionTime = metav1.NewTime(expired)
@@ -2919,8 +2924,9 @@ func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T)
}
// A plugin attached by someone else after the latch, another backup job or
// a restore, started the rollout before this job's write, which only finds
// it there: instances that rolled since the latch carry the plugin.
// the chart re-rendering a restored Cluster outside the platform flow,
// started the rollout before this job's write, which only finds it there:
// instances that rolled since the latch carry the plugin.
func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) {
oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second))
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
@@ -3452,6 +3458,41 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) {
}
}
// An in-place restore of the platform flow recreates the Cluster without the
// barman-cloud plugin, so a job held through it attaches the plugin again
// once released. That attach starts a rollout of its own, which the
// instances restored after the job's first rollout began have not done.
func TestReconcileCNPG_ReattachAfterRestoreRestartsTheRollout(t *testing.T) {
stampedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second)
j := newRolloutBackupJob(stampedAt.Add(-time.Hour))
j.Status.Conditions = []metav1.Condition{{
Type: backupCondPluginRolledOut,
Status: metav1.ConditionFalse,
Reason: "WaitingForPluginRollout",
LastTransitionTime: metav1.NewTime(stampedAt),
}}
f := newCNPGRolloutFixture(t, j, newRolloutCluster(cnpgClusterHealthyPhase),
newRolloutInstancePod(time.Now().Add(-2*time.Minute)))
restore := f.seedRestoreOfApp()
if got, _ := f.pass(); apimeta.FindStatusCondition(got.Status.Conditions, "Ready").Reason != ConditionReasonRestoreInProgress {
t.Fatalf("Ready = %+v, want the job held by the restore", apimeta.FindStatusCondition(got.Status.Conditions, "Ready"))
}
restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
if err := f.c.Status().Update(context.Background(), restore); err != nil {
t.Fatalf("finish the restore: %v", err)
}
attachAt := time.Now().Truncate(time.Second)
got, backups := f.pass()
if len(backups) != 0 {
t.Fatalf("cnpg.io/Backup created in the pass that attached the plugin to the restored Cluster: %+v", backups)
}
rollout := apimeta.FindStatusCondition(got.Status.Conditions, backupCondPluginRolledOut)
if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) {
t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt)
}
}
// A release older than the rollout, from a wait before the plugin was
// attached, does not shorten the rollout that began after it.
func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) {