mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(postgres): tolerate an attach record slightly ahead of the clock
An attach record in the future failed the backup job at once, because it would hold every instance as older than the plugin. A record a few minutes ahead is also what a leadership change between controllers whose clocks differ leaves behind after an attach that worked, and failing then throws away a backup that would have run. Within that margin the job now keeps waiting: the instances predate the record, so the rollout check holds the backup, and the deadline counted from the record is not spent by the wait. Only a record further ahead is taken as corrupt and fails the job. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
c69787ff49
commit
d37701a327
2 files changed
+62
-17
No files matched your search
@@ -233,6 +233,11 @@ const (
|
||||
// cannot be lost to a failed BackupJob status write after the attach.
|
||||
cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at"
|
||||
|
||||
// How far ahead of this controller's clock an attach record may be
|
||||
// before it is taken as corrupt rather than as written by a leader whose
|
||||
// clock ran ahead.
|
||||
cnpgAttachClockSkewTolerance = 5 * time.Minute
|
||||
|
||||
cnpgPodRoleLabel = "cnpg.io/podRole"
|
||||
cnpgPodRoleInstance = "instance"
|
||||
)
|
||||
@@ -499,20 +504,18 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
}
|
||||
// The Cluster's record covers an attach whose status write was lost.
|
||||
attachedAt := rollout.LastTransitionTime
|
||||
if attachment.attachedAt != "" {
|
||||
recorded, err := time.Parse(time.RFC3339, attachment.attachedAt)
|
||||
if recorded.After(time.Now()) {
|
||||
// Every instance would predate it, and the deadline would count from it.
|
||||
err = fmt.Errorf("%s is in the future", attachment.attachedAt)
|
||||
}
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
|
||||
"cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v",
|
||||
cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err))
|
||||
}
|
||||
if recorded.After(attachedAt.Time) {
|
||||
attachedAt = metav1.NewTime(recorded)
|
||||
}
|
||||
recorded, ok, err := cnpgRecordedAttach(attachment.attachedAt)
|
||||
if err == nil && ok && time.Until(recorded) > cnpgAttachClockSkewTolerance {
|
||||
// Every instance would predate it, and the deadline would count from it.
|
||||
err = fmt.Errorf("%s is in the future", attachment.attachedAt)
|
||||
}
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
|
||||
"cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v",
|
||||
cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err))
|
||||
}
|
||||
if ok && recorded.After(attachedAt.Time) {
|
||||
attachedAt = metav1.NewTime(recorded)
|
||||
}
|
||||
pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt)
|
||||
if err != nil {
|
||||
@@ -897,6 +900,19 @@ func (r *BackupJobReconciler) cnpgPluginRolloutPending(ctx context.Context, name
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// cnpgRecordedAttach parses a cnpgPluginAttachedAtAnnotation value; ok is
|
||||
// false for "", a Cluster that carries no record.
|
||||
func cnpgRecordedAttach(raw string) (recorded time.Time, ok bool, err error) {
|
||||
if raw == "" {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
recorded, err = time.Parse(time.RFC3339, raw)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
return recorded, true, nil
|
||||
}
|
||||
|
||||
func podReady(pod *corev1.Pod) bool {
|
||||
for _, c := range pod.Status.Conditions {
|
||||
if c.Type == corev1.PodReady {
|
||||
|
||||
@@ -3122,9 +3122,10 @@ func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) {
|
||||
}
|
||||
|
||||
// An attach record that does not parse cannot tell which instances rolled
|
||||
// with the plugin, so the job fails naming it instead of guessing. One in the
|
||||
// future would hold every instance as older than the plugin and push the
|
||||
// rollout deadline out by as far as it lies ahead, so it fails the same way.
|
||||
// with the plugin, so the job fails naming it instead of guessing. One further
|
||||
// ahead than controller clocks drift apart would hold every instance as older
|
||||
// than the plugin and push the rollout deadline out by as far as it lies
|
||||
// ahead, so it fails the same way.
|
||||
func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) {
|
||||
for _, tc := range []struct{ name, record string }{
|
||||
{name: "does not parse", record: "yesterday"},
|
||||
@@ -3151,6 +3152,34 @@ func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A record a little ahead of this controller's clock is what a leadership
|
||||
// change between controllers whose clocks differ leaves behind, so the job
|
||||
// waits for the clocks to agree rather than failing an attach that worked.
|
||||
func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) {
|
||||
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
|
||||
newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour)))
|
||||
if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil {
|
||||
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
|
||||
}
|
||||
live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))
|
||||
live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)}
|
||||
f.observeCNPG(live)
|
||||
|
||||
j, backups, res := f.passResult()
|
||||
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Fatalf("the job failed on a record a minute ahead: %s", j.Status.Message)
|
||||
}
|
||||
if len(backups) != 0 {
|
||||
t.Fatalf("a cnpg.io/Backup was created before the rollout could be judged: %+v", backups)
|
||||
}
|
||||
if res.RequeueAfter == 0 {
|
||||
t.Errorf("a waiting job is not requeued: %+v", res)
|
||||
}
|
||||
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" {
|
||||
t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready)
|
||||
}
|
||||
}
|
||||
|
||||
// A plugin attached by someone else after the latch, another backup job or
|
||||
// the chart re-rendering a restored Cluster outside the platform flow,
|
||||
// started the rollout before this job's write, which only finds it there:
|
||||
|
||||
Reference in new issue
Block a user