fix(postgres): keep the plugin attach time when its status write is lost

The rollout check took the attach time from the PluginRolledOut
condition on the BackupJob, which the pass that attaches the plugin
writes only after the Cluster apply went through. When that status
write is lost to a conflict or a controller restart, the next pass
finds the plugin attached and falls back to the latch written before
the attach. An instance recreated between the latch and the attach then
counts as one that rolled with the plugin, and the cnpg.io/Backup is
created against an instance that has no plugin sidecar; a latch older
than the deadline also fails a rollout that has only just begun.

The apply that attaches the plugin now records the time on the Cluster
itself, so the record cannot be lost apart from the attach. The rollout
check and its deadline use the later of the two times. A record that
does not parse fails the job rather than guess which instances rolled.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-10 00:08:41 +03:00
1 parent e5ccde82e2
commit 325d2b815c
2 files changed
+157 -11

No files matched your search

@@ -177,14 +177,15 @@ const (
// Cap on the wall-clock time a BackupJob spends before its
// cnpg.io/Backup exists on each of two waits. The plugin rollout is
// counted from the PluginRolledOut condition, or from the release
// counted from the attach, the later of the PluginRolledOut condition and
// the Cluster's cnpgPluginAttachedAtAnnotation, or from the release
// recorded in DestinationWriteHeld when that is later. Retrying the WAL
// archive destination write is counted from the first failure since the
// last successful write or the last wait for another backup or a
// restore. A Backup that exists is
// not capped: CloudNativePG keeps a healthy one pending while another
// backup of the same Cluster runs, or while it restarts one on another
// instance, so no fixed deadline tells those apart from a stuck one.
// restore. A Backup that exists is not capped: CloudNativePG keeps a
// healthy one pending while another backup of the same Cluster runs, or
// while it restarts one on another instance, so no fixed deadline tells
// those apart from a stuck one.
cnpgDefaultBackupDeadline = 30 * time.Minute
cnpgClusterHealthyPhase = "Cluster in healthy state"
@@ -195,9 +196,10 @@ const (
// instance pod spec (the plugin injects its sidecar), so CNPG rolls every
// instance, and a cnpg.io/Backup created meanwhile fails with "requested
// plugin is not available" or "instance manager was restarted during
// backup" and never recovers. False holds the Backup back; its
// LastTransitionTime is what the instance pods must postdate (see
// cnpgReasonAttachingPlugin and cnpgPluginRolloutPending).
// backup" and never recovers. False holds the Backup back; the instance
// pods must postdate its LastTransitionTime and the Cluster's
// cnpgPluginAttachedAtAnnotation (see cnpgReasonAttachingPlugin and
// cnpgPluginRolloutPending).
backupCondPluginRolledOut = "PluginRolledOut"
// Reason of the PluginRolledOut latch recorded before the write that
@@ -224,6 +226,12 @@ const (
// path, the endpoint URL and the name of the credentials Secret.
cnpgWrittenDestinationAnnotation = "backups.cozystack.io/cnpg-written-destination"
// Annotation on a cnpg.io/Cluster carrying, as RFC 3339, when a backup
// job attached the barman-cloud plugin. It is written by the same apply
// that attaches the plugin, so unlike the PluginRolledOut condition it
// cannot be lost to a failed BackupJob status write after the attach.
cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at"
cnpgPodRoleLabel = "cnpg.io/podRole"
cnpgPodRoleInstance = "instance"
)
@@ -487,12 +495,25 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
}
pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, rollout.LastTransitionTime)
// The Cluster's record covers an attach whose status write was lost.
attachedAt := rollout.LastTransitionTime
if attachment.attachedAt != "" {
recorded, err := time.Parse(time.RFC3339, attachment.attachedAt)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
"cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v",
cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err))
}
if recorded.After(attachedAt.Time) {
attachedAt = metav1.NewTime(recorded)
}
}
pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt)
if err != nil {
return ctrl.Result{}, err
}
if pending != "" {
deadlineFrom := rollout.LastTransitionTime
deadlineFrom := attachedAt
if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held != nil &&
held.LastTransitionTime.After(deadlineFrom.Time) {
deadlineFrom = held.LastTransitionTime
@@ -744,6 +765,7 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name
flavor: cnpgImageFlavor(existing.Spec.ImageName),
clusterStatus: existing.Status,
}
attachment.attachedAt = existing.Annotations[cnpgPluginAttachedAtAnnotation]
if live := currentBarmanServerName(existing); live != "" {
attachment.serverName = live
} else {
@@ -788,6 +810,15 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name
patch := newCNPGClusterPatch(namespace, clusterName)
patch.Spec.Plugins = []cnpgtypes.PluginConfiguration{buildBarmanPlugin(objStoreName, serverName)}
// Re-sent unchanged on later applies: the field manager owns it, and an
// apply that left it out would delete it.
stamp := existing.Annotations[cnpgPluginAttachedAtAnnotation]
if attachment.missing {
stamp = metav1.Now().UTC().Format(time.RFC3339)
}
if stamp != "" {
patch.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp}
}
if err := r.Patch(ctx, patch, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil {
return cnpgPluginAttachment{}, err
}
@@ -802,7 +833,10 @@ type cnpgPluginAttachment struct {
// flavor is the image flavor the live Cluster runs.
flavor string
// missing reports a Cluster that carried no barman-cloud plugin.
missing bool
missing bool
// attachedAt is the cnpgPluginAttachedAtAnnotation the Cluster carried,
// "" when no backup job attached the plugin.
attachedAt string
clusterStatus cnpgtypes.ClusterStatus
}
@@ -2924,6 +2924,118 @@ func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T)
}
}
// The status write that records the attach can be lost after the attach
// itself went through, by a conflict or a restart; an instance recreated
// between the latch and the attach still has to roll, and the rollout
// deadline still counts from the attach.
func TestReconcileCNPG_LostAttachRecordStillWaitsForTheRollout(t *testing.T) {
latchedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second)
oldPod := newRolloutInstancePod(latchedAt.Add(-time.Hour))
f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase), oldPod)
j, _ := f.pass() // latch
latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
if latch == nil {
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
}
latch.LastTransitionTime = metav1.NewTime(latchedAt)
if err := f.c.Status().Update(context.Background(), j); err != nil {
t.Fatalf("age the latch: %v", err)
}
f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(time.Minute)))
base := f.r.Client
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
return apierrors.NewConflict(backupsv1alpha1.GroupVersion.WithResource("backupjobs").GroupResource(), obj.GetName(), errors.New("the object has been modified"))
},
})
j = &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
t.Fatalf("the pass that attached the plugin reported no lost status write")
}
f.r.Client = base
attached := &cnpgtypes.Cluster{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil {
t.Fatalf("get Cluster: %v", err)
}
if len(attached.Spec.Plugins) == 0 {
t.Fatalf("the plugin was not attached")
}
live := newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)
live.Annotations = attached.Annotations
f.observeCNPG(live)
var backups []cnpgtypes.Backup
for range 3 {
if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
break
}
}
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the attach: %+v", backups)
}
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message)
}
}
// A later write to a Cluster that already carries the plugin re-sends the
// attach record as it is: moving it would make the instances that rolled
// with the plugin look older than the attach.
func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) {
const stamp = "2026-01-02T03:04:05Z"
cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))
cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp}
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster,
newRolloutInstancePod(time.Now().Add(-time.Hour)))
var sent []map[string]string
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if _, ok := obj.(*cnpgtypes.Cluster); ok {
sent = append(sent, obj.GetAnnotations())
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
if _, backups := f.pass(); len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups, want the one this pass creates", len(backups))
}
if len(sent) != 1 {
t.Fatalf("got %d Cluster writes, want 1", len(sent))
}
if got := sent[0][cnpgPluginAttachedAtAnnotation]; got != stamp {
t.Errorf("the write sent %s = %q, want the recorded %q", cnpgPluginAttachedAtAnnotation, got, stamp)
}
}
// An attach record that does not parse cannot tell which instances rolled
// with the plugin, so the job fails naming it instead of guessing.
func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) {
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour)))
if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil {
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
}
live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))
live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: "yesterday"}
f.observeCNPG(live)
j, backups := f.pass()
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created with an unreadable attach record: %+v", backups)
}
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgPluginAttachedAtAnnotation) {
t.Errorf("phase %q, message %q; want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgPluginAttachedAtAnnotation)
}
}
// A plugin attached by someone else after the latch, another backup job or
// the chart re-rendering a restored Cluster outside the platform flow,
// started the rollout before this job's write, which only finds it there: