diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index fc3fded4c..83ce9b41e 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -175,12 +175,13 @@ const ( // against an unreachable target reproduces the same FATAL. cnpgRecoveryMaxInspectPods = 5 - // Cap on the wall-clock time a BackupJob waits for the barman-cloud - // plugin rollout, counted from the PluginRolledOut condition or from the - // release recorded in DestinationWriteHeld, whichever is later, or retries - // writing the WAL archive destination, counted from the first failure - // since the last successful write or the last wait for another backup or - // a restore, before the cnpg.io/Backup exists. A Backup that exists is + // Cap on the wall-clock time a BackupJob spends before its + // cnpg.io/Backup exists on each of two waits. The plugin rollout is + // counted from the PluginRolledOut condition, or from the release + // recorded in DestinationWriteHeld when that is later. Retrying the WAL + // archive destination write is counted from the first failure since the + // last successful write or the last wait for another backup or a + // restore. A Backup that exists is // not capped: CloudNativePG keeps a healthy one pending while another // backup of the same Cluster runs, or while it restarts one on another // instance, so no fixed deadline tells those apart from a stuck one. @@ -200,8 +201,9 @@ const ( backupCondPluginRolledOut = "PluginRolledOut" // Reason of the PluginRolledOut latch recorded before the write that - // attaches the plugin. The write that attaches it replaces the latch with - // its own time, which retries can put long after the latch. + // attaches the plugin. Every write of this job that attaches it replaces + // the condition with its own time, which retries can put long after the + // latch. cnpgReasonAttachingPlugin = "AttachingPlugin" // Condition Type recorded while writing the WAL archive destination keeps @@ -454,9 +456,10 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } - if rollout.Reason == cnpgReasonAttachingPlugin && attachment.missing { - // Instances recreated between the latch and this write run - // without the plugin, and the rollout deadline starts here. A + if attachment.missing { + // This write attached the plugin, so its rollout starts here: + // instances recreated since the latch, or by a restore that + // re-rendered the Cluster without the plugin, run without it. A // plugin someone else attached since the latch keeps it: the // rollout that attach started may already be over. message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName) diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 1f0b88312..8f0035aa6 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2830,6 +2830,11 @@ func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) { if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" { t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond) } + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) rollout.LastTransitionTime = metav1.NewTime(expired) @@ -2919,8 +2924,9 @@ func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) } // A plugin attached by someone else after the latch, another backup job or -// a restore, started the rollout before this job's write, which only finds -// it there: instances that rolled since the latch carry the plugin. +// the chart re-rendering a restored Cluster outside the platform flow, +// started the rollout before this job's write, which only finds it there: +// instances that rolled since the latch carry the plugin. func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) { oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), @@ -3452,6 +3458,41 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) { } } +// An in-place restore of the platform flow recreates the Cluster without the +// barman-cloud plugin, so a job held through it attaches the plugin again +// once released. That attach starts a rollout of its own, which the +// instances restored after the job's first rollout began have not done. +func TestReconcileCNPG_ReattachAfterRestoreRestartsTheRollout(t *testing.T) { + stampedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(stampedAt), + }} + f := newCNPGRolloutFixture(t, j, newRolloutCluster(cnpgClusterHealthyPhase), + newRolloutInstancePod(time.Now().Add(-2*time.Minute))) + restore := f.seedRestoreOfApp() + if got, _ := f.pass(); apimeta.FindStatusCondition(got.Status.Conditions, "Ready").Reason != ConditionReasonRestoreInProgress { + t.Fatalf("Ready = %+v, want the job held by the restore", apimeta.FindStatusCondition(got.Status.Conditions, "Ready")) + } + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + + attachAt := time.Now().Truncate(time.Second) + got, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("cnpg.io/Backup created in the pass that attached the plugin to the restored Cluster: %+v", backups) + } + rollout := apimeta.FindStatusCondition(got.Status.Conditions, backupCondPluginRolledOut) + if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { + t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) + } +} + // A release older than the rollout, from a wait before the plugin was // attached, does not shorten the rollout that began after it. func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) {