mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(postgres): time the plugin rollout from the write that attached it
The PluginRolledOut latch is recorded before the Cluster write that attaches the barman-cloud plugin, and its time was used both as the start of the rollout deadline and as the age every instance pod must exceed to count as rolled. Now that a failing write is retried for up to the backup deadline, the attach can come long after the latch: the deadline was then spent before the rollout began, and a pod recreated between the latch and the attach, without the plugin sidecar, passed as rolled, so a Backup was created that CloudNativePG fails for good. The latch now carries its own reason, and the write that finds the plugin missing and attaches it re-stamps the condition, once, with the time it was attempted. A plugin someone else attached after the latch, another backup job or a restore, keeps the latch time: the rollout it started can be over by the time this job writes, and its instances would otherwise never count as rolled. Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io> Assisted-by: LLM
This commit is contained in:
1 parent
570fcec725
commit
567f07d708
2 files changed
+140
No files matched your search
@@ -197,6 +197,11 @@ const (
|
||||
// instance pods must postdate (see cnpgPluginRolloutPending).
|
||||
backupCondPluginRolledOut = "PluginRolledOut"
|
||||
|
||||
// Reason of the PluginRolledOut latch recorded before the write that
|
||||
// attaches the plugin. The write that attaches it replaces the latch with
|
||||
// its own time, which retries can put long after the latch.
|
||||
cnpgReasonAttachingPlugin = "AttachingPlugin"
|
||||
|
||||
// Condition Type recorded while writing the WAL archive destination keeps
|
||||
// failing. Its LastTransitionTime marks the first failure, so the retry
|
||||
// budget is not spent by time the job waited for another backup of the
|
||||
@@ -356,6 +361,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
}
|
||||
}
|
||||
|
||||
attemptedAt := metav1.Now()
|
||||
attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination)
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
@@ -419,9 +425,34 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: backupCondPluginRolledOut,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: cnpgReasonAttachingPlugin,
|
||||
Message: message,
|
||||
})
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "WaitingForPluginRollout",
|
||||
Message: message,
|
||||
})
|
||||
if err := r.Status().Update(ctx, j); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
|
||||
}
|
||||
if rollout.Reason == cnpgReasonAttachingPlugin && attachment.missing {
|
||||
// Instances recreated between the latch and this write run
|
||||
// without the plugin, and the rollout deadline starts here. A
|
||||
// plugin someone else attached since the latch keeps it: the
|
||||
// rollout that attach started may already be over.
|
||||
message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName)
|
||||
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondPluginRolledOut)
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: backupCondPluginRolledOut,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "WaitingForPluginRollout",
|
||||
Message: message,
|
||||
LastTransitionTime: attemptedAt,
|
||||
})
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: metav1.ConditionFalse,
|
||||
|
||||
@@ -2814,6 +2814,115 @@ func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The rollout is measured from the write that attached the plugin, not from
|
||||
// the latch recorded before it: retries of a failing write can lie between
|
||||
// the two, and an instance recreated in that time runs without the plugin.
|
||||
func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
// podBetween recreates the instance after the latch and before the
|
||||
// attach; otherwise it predates both.
|
||||
podBetween bool
|
||||
}{
|
||||
{name: "an instance recreated before the attach still has to roll", podBetween: true},
|
||||
{name: "retries before the attach do not spend the rollout deadline"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second)
|
||||
oldPod := newRolloutInstancePod(expired.Add(-time.Hour))
|
||||
f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)),
|
||||
newRolloutCluster(cnpgClusterHealthyPhase), oldPod)
|
||||
|
||||
j, _ := f.pass() // latch
|
||||
latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
|
||||
if latch == nil {
|
||||
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
|
||||
}
|
||||
latch.LastTransitionTime = metav1.NewTime(expired)
|
||||
if err := f.c.Status().Update(context.Background(), j); err != nil {
|
||||
t.Fatalf("age the latch: %v", err)
|
||||
}
|
||||
|
||||
failing := true
|
||||
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
|
||||
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
||||
if _, ok := obj.(*cnpgtypes.Cluster); ok && failing {
|
||||
return apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io"`))
|
||||
}
|
||||
return cl.Patch(ctx, obj, patch, opts...)
|
||||
},
|
||||
})
|
||||
if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Fatalf("a failed attach failed the job: %s", j.Status.Message)
|
||||
}
|
||||
if tc.podBetween {
|
||||
f.replacePod(oldPod, newRolloutInstancePod(expired.Add(time.Minute)))
|
||||
}
|
||||
|
||||
failing = false
|
||||
attachAt := time.Now().Truncate(time.Second)
|
||||
j, backups := f.pass()
|
||||
if len(backups) != 0 {
|
||||
t.Fatalf("a cnpg.io/Backup was created in the pass that attached the plugin: %+v", backups)
|
||||
}
|
||||
rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
|
||||
if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) {
|
||||
t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt)
|
||||
}
|
||||
|
||||
attached := &cnpgtypes.Cluster{}
|
||||
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil {
|
||||
t.Fatalf("get Cluster: %v", err)
|
||||
}
|
||||
f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...))
|
||||
j, backups = f.pass()
|
||||
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Fatalf("a rollout that began with the attach failed: %s", j.Status.Message)
|
||||
}
|
||||
if len(backups) != 0 {
|
||||
t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups)
|
||||
}
|
||||
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || !strings.Contains(ready.Message, "predates") {
|
||||
t.Errorf("Ready = %+v, want the instance that predates the plugin", ready)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A plugin attached by someone else after the latch, another backup job or
|
||||
// a restore, started the rollout before this job's write, which only finds
|
||||
// it there: instances that rolled since the latch carry the plugin.
|
||||
func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) {
|
||||
oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second))
|
||||
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
|
||||
newRolloutCluster(cnpgClusterHealthyPhase), oldPod)
|
||||
|
||||
j, _ := f.pass() // latch
|
||||
latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
|
||||
if latch == nil {
|
||||
t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions)
|
||||
}
|
||||
latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second)
|
||||
latch.LastTransitionTime = metav1.NewTime(latchedAt)
|
||||
if err := f.c.Status().Update(context.Background(), j); err != nil {
|
||||
t.Fatalf("age the latch: %v", err)
|
||||
}
|
||||
f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)))
|
||||
f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(5*time.Minute)))
|
||||
|
||||
var backups []cnpgtypes.Backup
|
||||
for range 3 {
|
||||
if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(backups) != 1 {
|
||||
t.Fatalf("got %d cnpg.io/Backups against instances that rolled with the plugin; Ready = %+v",
|
||||
len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
|
||||
}
|
||||
}
|
||||
|
||||
// Waiting for another backup resets the retry budget of the destination
|
||||
// write, so the queue does not spend it.
|
||||
func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user