mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(postgres): give a held backup job its plugin rollout deadline back
A job that latched the barman-cloud plugin rollout and then waited for another backup of the application, or for a restore of it, had that wait charged to the rollout deadline. Released after more than the deadline, it failed on the first instance that was not ready yet with "rollout did not finish", although no rollout of its own had been running while it waited. The hold is now recorded in a DestinationWriteHeld condition, and its release time restarts the rollout deadline when it is later than the rollout's start. The instance pods are still measured against the rollout's start: one that rolled while the job was held carries the plugin and counts. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
8061e2dbd9
commit
32ca678077
2 files changed
+205
-2
No files matched your search
@@ -176,7 +176,8 @@ const (
|
||||
cnpgRecoveryMaxInspectPods = 5
|
||||
|
||||
// Cap on the wall-clock time a BackupJob waits for the barman-cloud
|
||||
// plugin rollout, counted from the PluginRolledOut condition, or retries
|
||||
// plugin rollout, counted from the PluginRolledOut condition or from the
|
||||
// release recorded in DestinationWriteHeld, whichever is later, or retries
|
||||
// writing the WAL archive destination, counted from the first failure
|
||||
// since the last successful write or the last wait for another backup or
|
||||
// a restore, before the cnpg.io/Backup exists. A Backup that exists is
|
||||
@@ -209,6 +210,13 @@ const (
|
||||
// app, or for a restore of it, before it tried.
|
||||
backupCondDestinationApplied = "DestinationApplied"
|
||||
|
||||
// Condition Type recorded while the job is held off writing the WAL
|
||||
// archive destination, by another backup of the app in flight or by a
|
||||
// restore of it, and set False when released. The release time restarts
|
||||
// the plugin rollout deadline, which that wait must not spend; the
|
||||
// instance pods still have to postdate the rollout's own start.
|
||||
backupCondDestinationWriteHeld = "DestinationWriteHeld"
|
||||
|
||||
cnpgPodRoleLabel = "cnpg.io/podRole"
|
||||
cnpgPodRoleInstance = "instance"
|
||||
)
|
||||
@@ -349,6 +357,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
}
|
||||
if restore != "" {
|
||||
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld(ConditionReasonRestoreInProgress))
|
||||
return r.holdForRestore(ctx, j, restore)
|
||||
}
|
||||
holder, err := r.otherCNPGBackupInFlight(ctx, j, clusterName)
|
||||
@@ -357,9 +366,18 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
}
|
||||
if holder != "" {
|
||||
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForOtherBackup"))
|
||||
return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf(
|
||||
"waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder))
|
||||
}
|
||||
if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil {
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: backupCondDestinationWriteHeld,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "Released",
|
||||
Message: "nothing holds the WAL archive destination write any more",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
attemptedAt := metav1.Now()
|
||||
@@ -466,7 +484,12 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if pending != "" {
|
||||
if cnpgBackupDeadlineExceeded(&rollout.LastTransitionTime) {
|
||||
deadlineFrom := rollout.LastTransitionTime
|
||||
if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held != nil &&
|
||||
held.LastTransitionTime.After(deadlineFrom.Time) {
|
||||
deadlineFrom = held.LastTransitionTime
|
||||
}
|
||||
if cnpgBackupDeadlineExceeded(&deadlineFrom) {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
|
||||
"barman-cloud plugin rollout did not finish within %s: %s", cnpgDefaultBackupDeadline, pending))
|
||||
}
|
||||
@@ -933,6 +956,15 @@ func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *ba
|
||||
return holders[0], nil
|
||||
}
|
||||
|
||||
func cnpgDestinationWriteHeld(reason string) metav1.Condition {
|
||||
return metav1.Condition{
|
||||
Type: backupCondDestinationWriteHeld,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: reason,
|
||||
Message: "held off writing the WAL archive destination",
|
||||
}
|
||||
}
|
||||
|
||||
// waitCNPGBackupJob requeues the BackupJob and records why it waits in its
|
||||
// Ready condition, writing the status only when the conditions changed.
|
||||
func (r *BackupJobReconciler) waitCNPGBackupJob(ctx context.Context, j *backupsv1alpha1.BackupJob, reason, message string) (ctrl.Result, error) {
|
||||
|
||||
@@ -3447,6 +3447,41 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) {
|
||||
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); cond != nil {
|
||||
t.Errorf("%s condition = %+v on a Cluster the driver did not roll", backupCondPluginRolledOut, cond)
|
||||
}
|
||||
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); cond != nil {
|
||||
t.Errorf("%s condition = %+v on a job nothing held", backupCondDestinationWriteHeld, cond)
|
||||
}
|
||||
}
|
||||
|
||||
// A release older than the rollout, from a wait before the plugin was
|
||||
// attached, does not shorten the rollout that began after it.
|
||||
func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) {
|
||||
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
|
||||
j := newRolloutBackupJob(expired.Add(-time.Hour))
|
||||
j.Status.Conditions = []metav1.Condition{
|
||||
{
|
||||
Type: backupCondDestinationWriteHeld,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "Released",
|
||||
LastTransitionTime: metav1.NewTime(expired),
|
||||
},
|
||||
{
|
||||
Type: backupCondPluginRolledOut,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "WaitingForPluginRollout",
|
||||
LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)),
|
||||
},
|
||||
}
|
||||
f := newCNPGRolloutFixture(t, j,
|
||||
newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)),
|
||||
newRolloutInstancePod(expired.Add(-time.Hour)))
|
||||
|
||||
got, backups := f.pass()
|
||||
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Fatalf("a rollout that began a minute ago failed on an older release: %s", got.Status.Message)
|
||||
}
|
||||
if len(backups) != 0 {
|
||||
t.Fatalf("a cnpg.io/Backup was created against an unfinished rollout: %+v", backups)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileCNPG_PluginRolloutWaitIsBounded: a rollout that never ends
|
||||
@@ -3476,6 +3511,142 @@ func TestReconcileCNPG_PluginRolloutWaitIsBounded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A job whose plugin rollout started, and which then waited for another
|
||||
// backup of the app or for a restore of it, gets the rollout deadline back
|
||||
// from the moment it was released: that wait is not a rollout of its own.
|
||||
// The instance pods still count as rolled from the rollout's own start.
|
||||
func TestReconcileCNPG_HeldJobRolloutDeadlineCountsFromRelease(t *testing.T) {
|
||||
type hold struct {
|
||||
name string
|
||||
seed func(f *cnpgRolloutFixture) (release func())
|
||||
waiting string
|
||||
}
|
||||
holds := []hold{
|
||||
{
|
||||
name: "another backup",
|
||||
waiting: "WaitingForOtherBackup",
|
||||
seed: func(f *cnpgRolloutFixture) func() {
|
||||
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
|
||||
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
|
||||
return func() {
|
||||
held := &cnpgtypes.Backup{}
|
||||
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil {
|
||||
f.t.Fatalf("get holder Backup: %v", err)
|
||||
}
|
||||
held.Status.Phase = cnpgBackupPhaseComplete
|
||||
if err := f.c.Update(context.Background(), held); err != nil {
|
||||
f.t.Fatalf("complete holder Backup: %v", err)
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "a restore",
|
||||
waiting: ConditionReasonRestoreInProgress,
|
||||
seed: func(f *cnpgRolloutFixture) func() {
|
||||
restore := f.seedRestoreOfApp()
|
||||
return func() {
|
||||
restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
|
||||
if err := f.c.Status().Update(context.Background(), restore); err != nil {
|
||||
f.t.Fatalf("finish the restore: %v", err)
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
stampedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second)
|
||||
newHeldRollout := func(t *testing.T, h hold, cluster *cnpgtypes.Cluster, pod *corev1.Pod) *cnpgRolloutFixture {
|
||||
t.Helper()
|
||||
j := newRolloutBackupJob(stampedAt.Add(-time.Hour))
|
||||
j.Status.Conditions = []metav1.Condition{{
|
||||
Type: backupCondPluginRolledOut,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "WaitingForPluginRollout",
|
||||
LastTransitionTime: metav1.NewTime(stampedAt),
|
||||
}}
|
||||
f := newCNPGRolloutFixture(t, j, cluster, pod)
|
||||
release := h.seed(f)
|
||||
got, backups := f.pass()
|
||||
for _, b := range backups {
|
||||
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
|
||||
t.Fatalf("a held job created cnpg.io/Backup %s", b.Name)
|
||||
}
|
||||
}
|
||||
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Fatalf("a held job did not wait: %s", got.Status.Message)
|
||||
}
|
||||
if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready == nil || ready.Reason != h.waiting {
|
||||
t.Fatalf("Ready = %+v, want %s", ready, h.waiting)
|
||||
}
|
||||
release()
|
||||
return f
|
||||
}
|
||||
|
||||
for _, h := range holds {
|
||||
t.Run(h.name+": the rollout is still waited for after the release", func(t *testing.T) {
|
||||
f := newHeldRollout(t, h,
|
||||
newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)),
|
||||
newRolloutInstancePod(stampedAt.Add(-time.Hour)))
|
||||
|
||||
releasedAt := time.Now().Truncate(time.Second)
|
||||
j, backups := f.pass()
|
||||
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
t.Fatalf("time held by %s was charged to the rollout: %s", h.name, j.Status.Message)
|
||||
}
|
||||
for _, b := range backups {
|
||||
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
|
||||
t.Fatalf("cnpg.io/Backup %s was created against an unfinished rollout", b.Name)
|
||||
}
|
||||
}
|
||||
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" {
|
||||
t.Fatalf("Ready = %+v, want WaitingForPluginRollout", ready)
|
||||
}
|
||||
released := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld)
|
||||
if released == nil || released.Status != metav1.ConditionFalse || released.LastTransitionTime.Time.Before(releasedAt) {
|
||||
t.Fatalf("%s = %+v, want False since the release at %s", backupCondDestinationWriteHeld, released, releasedAt)
|
||||
}
|
||||
if rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); rollout == nil || !rollout.LastTransitionTime.Time.Equal(stampedAt) {
|
||||
t.Errorf("%s = %+v, want its stamp kept at %s", backupCondPluginRolledOut, rollout, stampedAt)
|
||||
}
|
||||
|
||||
released.LastTransitionTime = metav1.NewTime(stampedAt)
|
||||
if err := f.c.Status().Update(context.Background(), j); err != nil {
|
||||
t.Fatalf("age the release: %v", err)
|
||||
}
|
||||
j, _ = f.pass()
|
||||
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") {
|
||||
t.Fatalf("phase = %q (%s), want Failed once the rollout outlived the deadline after the release", j.Status.Phase, j.Status.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(h.name+": an instance that rolled while the job was held counts", func(t *testing.T) {
|
||||
f := newHeldRollout(t, h,
|
||||
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
|
||||
newRolloutInstancePod(stampedAt.Add(time.Minute)))
|
||||
|
||||
var j *backupsv1alpha1.BackupJob
|
||||
own := 0
|
||||
for range 3 {
|
||||
var backups []cnpgtypes.Backup
|
||||
j, backups = f.pass()
|
||||
own = 0
|
||||
for _, b := range backups {
|
||||
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
|
||||
own++
|
||||
}
|
||||
}
|
||||
if own != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
break
|
||||
}
|
||||
}
|
||||
if own != 1 {
|
||||
t.Fatalf("got %d cnpg.io/Backups of the job against an instance that rolled after the rollout began; Ready = %+v, phase %q (%s)",
|
||||
own, apimeta.FindStatusCondition(j.Status.Conditions, "Ready"), j.Status.Phase, j.Status.Message)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline covers the
|
||||
// restore-path blocker: a RestoreJob whose referenced CNPG Strategy CR never
|
||||
// appears must NOT requeue forever. requeueRestoreStrategyNotReady gates
|
||||
|
||||
Reference in new issue
Block a user