fix(postgres): tolerate an attach record slightly ahead of the clock

An attach record in the future failed the backup job at once, because
it would hold every instance as older than the plugin. A record a few
minutes ahead is also what a leadership change between controllers
whose clocks differ leaves behind after an attach that worked, and
failing then throws away a backup that would have run. Within that
margin the job now keeps waiting: the instances predate the record, so
the rollout check holds the backup, and the deadline counted from the
record is not spent by the wait. Only a record further ahead is taken
as corrupt and fails the job.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-10 00:08:41 +03:00
1 parent c69787ff49
commit d37701a327
2 files changed
+62 -17

No files matched your search

@@ -233,6 +233,11 @@ const (
// cannot be lost to a failed BackupJob status write after the attach.
cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at"
// How far ahead of this controller's clock an attach record may be
// before it is taken as corrupt rather than as written by a leader whose
// clock ran ahead.
cnpgAttachClockSkewTolerance = 5 * time.Minute
cnpgPodRoleLabel = "cnpg.io/podRole"
cnpgPodRoleInstance = "instance"
)
@@ -499,20 +504,18 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
// The Cluster's record covers an attach whose status write was lost.
attachedAt := rollout.LastTransitionTime
if attachment.attachedAt != "" {
recorded, err := time.Parse(time.RFC3339, attachment.attachedAt)
if recorded.After(time.Now()) {
// Every instance would predate it, and the deadline would count from it.
err = fmt.Errorf("%s is in the future", attachment.attachedAt)
}
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
"cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v",
cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err))
}
if recorded.After(attachedAt.Time) {
attachedAt = metav1.NewTime(recorded)
}
recorded, ok, err := cnpgRecordedAttach(attachment.attachedAt)
if err == nil && ok && time.Until(recorded) > cnpgAttachClockSkewTolerance {
// Every instance would predate it, and the deadline would count from it.
err = fmt.Errorf("%s is in the future", attachment.attachedAt)
}
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
"cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v",
cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err))
}
if ok && recorded.After(attachedAt.Time) {
attachedAt = metav1.NewTime(recorded)
}
pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt)
if err != nil {
@@ -897,6 +900,19 @@ func (r *BackupJobReconciler) cnpgPluginRolloutPending(ctx context.Context, name
return "", nil
}
// cnpgRecordedAttach parses a cnpgPluginAttachedAtAnnotation value; ok is
// false for "", a Cluster that carries no record.
func cnpgRecordedAttach(raw string) (recorded time.Time, ok bool, err error) {
if raw == "" {
return time.Time{}, false, nil
}
recorded, err = time.Parse(time.RFC3339, raw)
if err != nil {
return time.Time{}, false, err
}
return recorded, true, nil
}
func podReady(pod *corev1.Pod) bool {
for _, c := range pod.Status.Conditions {
if c.Type == corev1.PodReady {
@@ -3122,9 +3122,10 @@ func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) {
}
// An attach record that does not parse cannot tell which instances rolled
// with the plugin, so the job fails naming it instead of guessing. One in the
// future would hold every instance as older than the plugin and push the
// rollout deadline out by as far as it lies ahead, so it fails the same way.
// with the plugin, so the job fails naming it instead of guessing. One further
// ahead than controller clocks drift apart would hold every instance as older
// than the plugin and push the rollout deadline out by as far as it lies
// ahead, so it fails the same way.
func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) {
for _, tc := range []struct{ name, record string }{
{name: "does not parse", record: "yesterday"},
@@ -3151,6 +3152,34 @@ func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) {
}
}
// A record a little ahead of this controller's clock is what a leadership
// change between controllers whose clocks differ leaves behind, so the job
// waits for the clocks to agree rather than failing an attach that worked.
func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) {
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour)))
if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil {
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
}
live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))
live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)}
f.observeCNPG(live)
j, backups, res := f.passResult()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("the job failed on a record a minute ahead: %s", j.Status.Message)
}
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created before the rollout could be judged: %+v", backups)
}
if res.RequeueAfter == 0 {
t.Errorf("a waiting job is not requeued: %+v", res)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" {
t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready)
}
}
// A plugin attached by someone else after the latch, another backup job or
// the chart re-rendering a restored Cluster outside the platform flow,
// started the rollout before this job's write, which only finds it there: