diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 3eb3421d6..5869103ca 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -177,14 +177,15 @@ const ( // Cap on the wall-clock time a BackupJob spends before its // cnpg.io/Backup exists on each of two waits. The plugin rollout is - // counted from the PluginRolledOut condition, or from the release + // counted from the attach, the later of the PluginRolledOut condition and + // the Cluster's cnpgPluginAttachedAtAnnotation, or from the release // recorded in DestinationWriteHeld when that is later. Retrying the WAL // archive destination write is counted from the first failure since the // last successful write or the last wait for another backup or a - // restore. A Backup that exists is - // not capped: CloudNativePG keeps a healthy one pending while another - // backup of the same Cluster runs, or while it restarts one on another - // instance, so no fixed deadline tells those apart from a stuck one. + // restore. A Backup that exists is not capped: CloudNativePG keeps a + // healthy one pending while another backup of the same Cluster runs, or + // while it restarts one on another instance, so no fixed deadline tells + // those apart from a stuck one. cnpgDefaultBackupDeadline = 30 * time.Minute cnpgClusterHealthyPhase = "Cluster in healthy state" @@ -195,9 +196,10 @@ const ( // instance pod spec (the plugin injects its sidecar), so CNPG rolls every // instance, and a cnpg.io/Backup created meanwhile fails with "requested // plugin is not available" or "instance manager was restarted during - // backup" and never recovers. False holds the Backup back; its - // LastTransitionTime is what the instance pods must postdate (see - // cnpgReasonAttachingPlugin and cnpgPluginRolloutPending). + // backup" and never recovers. False holds the Backup back; the instance + // pods must postdate its LastTransitionTime and the Cluster's + // cnpgPluginAttachedAtAnnotation (see cnpgReasonAttachingPlugin and + // cnpgPluginRolloutPending). backupCondPluginRolledOut = "PluginRolledOut" // Reason of the PluginRolledOut latch recorded before the write that @@ -224,6 +226,12 @@ const ( // path, the endpoint URL and the name of the credentials Secret. cnpgWrittenDestinationAnnotation = "backups.cozystack.io/cnpg-written-destination" + // Annotation on a cnpg.io/Cluster carrying, as RFC 3339, when a backup + // job attached the barman-cloud plugin. It is written by the same apply + // that attaches the plugin, so unlike the PluginRolledOut condition it + // cannot be lost to a failed BackupJob status write after the attach. + cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at" + cnpgPodRoleLabel = "cnpg.io/podRole" cnpgPodRoleInstance = "instance" ) @@ -487,12 +495,25 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } - pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, rollout.LastTransitionTime) + // The Cluster's record covers an attach whose status write was lost. + attachedAt := rollout.LastTransitionTime + if attachment.attachedAt != "" { + recorded, err := time.Parse(time.RFC3339, attachment.attachedAt) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf( + "cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v", + cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err)) + } + if recorded.After(attachedAt.Time) { + attachedAt = metav1.NewTime(recorded) + } + } + pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt) if err != nil { return ctrl.Result{}, err } if pending != "" { - deadlineFrom := rollout.LastTransitionTime + deadlineFrom := attachedAt if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held != nil && held.LastTransitionTime.After(deadlineFrom.Time) { deadlineFrom = held.LastTransitionTime @@ -744,6 +765,7 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name flavor: cnpgImageFlavor(existing.Spec.ImageName), clusterStatus: existing.Status, } + attachment.attachedAt = existing.Annotations[cnpgPluginAttachedAtAnnotation] if live := currentBarmanServerName(existing); live != "" { attachment.serverName = live } else { @@ -788,6 +810,15 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name patch := newCNPGClusterPatch(namespace, clusterName) patch.Spec.Plugins = []cnpgtypes.PluginConfiguration{buildBarmanPlugin(objStoreName, serverName)} + // Re-sent unchanged on later applies: the field manager owns it, and an + // apply that left it out would delete it. + stamp := existing.Annotations[cnpgPluginAttachedAtAnnotation] + if attachment.missing { + stamp = metav1.Now().UTC().Format(time.RFC3339) + } + if stamp != "" { + patch.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} + } if err := r.Patch(ctx, patch, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return cnpgPluginAttachment{}, err } @@ -802,7 +833,10 @@ type cnpgPluginAttachment struct { // flavor is the image flavor the live Cluster runs. flavor string // missing reports a Cluster that carried no barman-cloud plugin. - missing bool + missing bool + // attachedAt is the cnpgPluginAttachedAtAnnotation the Cluster carried, + // "" when no backup job attached the plugin. + attachedAt string clusterStatus cnpgtypes.ClusterStatus } diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index de525f0a7..255be9a27 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2924,6 +2924,118 @@ func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) } } +// The status write that records the attach can be lost after the attach +// itself went through, by a conflict or a restart; an instance recreated +// between the latch and the attach still has to roll, and the rollout +// deadline still counts from the attach. +func TestReconcileCNPG_LostAttachRecordStillWaitsForTheRollout(t *testing.T) { + latchedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + oldPod := newRolloutInstancePod(latchedAt.Add(-time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(time.Minute))) + + base := f.r.Client + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return apierrors.NewConflict(backupsv1alpha1.GroupVersion.WithResource("backupjobs").GroupResource(), obj.GetName(), errors.New("the object has been modified")) + }, + }) + j = &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("the pass that attached the plugin reported no lost status write") + } + f.r.Client = base + + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + if len(attached.Spec.Plugins) == 0 { + t.Fatalf("the plugin was not attached") + } + live := newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...) + live.Annotations = attached.Annotations + f.observeCNPG(live) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the attach: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) + } +} + +// A later write to a Cluster that already carries the plugin re-sends the +// attach record as it is: moving it would make the instances that rolled +// with the plugin look older than the attach. +func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) { + const stamp = "2026-01-02T03:04:05Z" + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(time.Now().Add(-time.Hour))) + + var sent []map[string]string + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok { + sent = append(sent, obj.GetAnnotations()) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if _, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want the one this pass creates", len(backups)) + } + if len(sent) != 1 { + t.Fatalf("got %d Cluster writes, want 1", len(sent)) + } + if got := sent[0][cnpgPluginAttachedAtAnnotation]; got != stamp { + t.Errorf("the write sent %s = %q, want the recorded %q", cnpgPluginAttachedAtAnnotation, got, stamp) + } +} + +// An attach record that does not parse cannot tell which instances rolled +// with the plugin, so the job fails naming it instead of guessing. +func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: "yesterday"} + f.observeCNPG(live) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created with an unreadable attach record: %+v", backups) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgPluginAttachedAtAnnotation) { + t.Errorf("phase %q, message %q; want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgPluginAttachedAtAnnotation) + } +} + // A plugin attached by someone else after the latch, another backup job or // the chart re-rendering a restored Cluster outside the platform flow, // started the rollout before this job's write, which only finds it there: