diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index a4bb0a3b9..a9eb6fabc 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -233,6 +233,11 @@ const ( // cannot be lost to a failed BackupJob status write after the attach. cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at" + // How far ahead of this controller's clock an attach record may be + // before it is taken as corrupt rather than as written by a leader whose + // clock ran ahead. + cnpgAttachClockSkewTolerance = 5 * time.Minute + cnpgPodRoleLabel = "cnpg.io/podRole" cnpgPodRoleInstance = "instance" ) @@ -499,20 +504,18 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } // The Cluster's record covers an attach whose status write was lost. attachedAt := rollout.LastTransitionTime - if attachment.attachedAt != "" { - recorded, err := time.Parse(time.RFC3339, attachment.attachedAt) - if recorded.After(time.Now()) { - // Every instance would predate it, and the deadline would count from it. - err = fmt.Errorf("%s is in the future", attachment.attachedAt) - } - if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf( - "cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v", - cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err)) - } - if recorded.After(attachedAt.Time) { - attachedAt = metav1.NewTime(recorded) - } + recorded, ok, err := cnpgRecordedAttach(attachment.attachedAt) + if err == nil && ok && time.Until(recorded) > cnpgAttachClockSkewTolerance { + // Every instance would predate it, and the deadline would count from it. + err = fmt.Errorf("%s is in the future", attachment.attachedAt) + } + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf( + "cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v", + cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err)) + } + if ok && recorded.After(attachedAt.Time) { + attachedAt = metav1.NewTime(recorded) } pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt) if err != nil { @@ -897,6 +900,19 @@ func (r *BackupJobReconciler) cnpgPluginRolloutPending(ctx context.Context, name return "", nil } +// cnpgRecordedAttach parses a cnpgPluginAttachedAtAnnotation value; ok is +// false for "", a Cluster that carries no record. +func cnpgRecordedAttach(raw string) (recorded time.Time, ok bool, err error) { + if raw == "" { + return time.Time{}, false, nil + } + recorded, err = time.Parse(time.RFC3339, raw) + if err != nil { + return time.Time{}, false, err + } + return recorded, true, nil +} + func podReady(pod *corev1.Pod) bool { for _, c := range pod.Status.Conditions { if c.Type == corev1.PodReady { diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 25a72b746..85afa0c65 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -3122,9 +3122,10 @@ func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) { } // An attach record that does not parse cannot tell which instances rolled -// with the plugin, so the job fails naming it instead of guessing. One in the -// future would hold every instance as older than the plugin and push the -// rollout deadline out by as far as it lies ahead, so it fails the same way. +// with the plugin, so the job fails naming it instead of guessing. One further +// ahead than controller clocks drift apart would hold every instance as older +// than the plugin and push the rollout deadline out by as far as it lies +// ahead, so it fails the same way. func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) { for _, tc := range []struct{ name, record string }{ {name: "does not parse", record: "yesterday"}, @@ -3151,6 +3152,34 @@ func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) { } } +// A record a little ahead of this controller's clock is what a leadership +// change between controllers whose clocks differ leaves behind, so the job +// waits for the clocks to agree rather than failing an attach that worked. +func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed on a record a minute ahead: %s", j.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created before the rollout could be judged: %+v", backups) + } + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) + } +} + // A plugin attached by someone else after the latch, another backup job or // the chart re-rendering a restored Cluster outside the platform flow, // started the rollout before this job's write, which only finds it there: