fix(postgres): stop backup jobs from fighting over the WAL archive destination

The ObjectStore a Postgres Cluster archives its WAL to is shared by
every backup of the app, yet each running BackupJob server-side applied
its own strategy's destination onto it every five seconds with forced
ownership. Two BackupJobs of one app with different BackupClasses
flipped the destination back and forth until the healthy backup failed
too, and a webhook or apiserver blip during that apply failed the
BackupJob at once.

A BackupJob now writes the destination only before its cnpg.io/Backup
exists, and waits, with a WaitingForOtherBackup condition, while
another running BackupJob of the app has a Backup in flight. The check
reads the informer cache, so jobs released together, or one created
while another's Backup is not yet cached, can still both proceed; an
atomic per-application claim is a separate change.

Write errors that cannot succeed on retry (invalid, bad request,
forbidden) still fail the job at once. Any other is retried for the
backup deadline counted from its first failure since the job last left
the queue, recorded in a DestinationApplied condition. Once the Backup
exists the Cluster is only read, and a failed read is returned for a
controller retry instead of failing the running backup. The plugin
rollout deadline counts from the attach rather than from the job's
start, so queueing before the attach no longer shortens it.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-09 23:52:25 +03:00
1 parent b53ca1fede
commit 9cc9a3b48d
4 files changed
+563 -31

No files matched your search

@@ -315,14 +315,6 @@ func (r *BackupJobReconciler) handleProjectionError(ctx context.Context, j *back
}
// markBackupJobFailed records a terminal Failed phase on the BackupJob.
//
// Coupling note: a failure that fires before reconcileCNPG's StartedAt block
// has set backupJob.Status.StartedAt leaves StartedAt nil. The CNPG path's
// cnpgDefaultBackupDeadline check only fires once StartedAt is set, so an
// early-failure retry that reaches the StartedAt block restarts the deadline
// budget from the retry's StartedAt - intentional, since retries are
// user-driven and a fresh budget is what users expect after correcting the
// cause of the previous failure.
func (r *BackupJobReconciler) markBackupJobFailed(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, message string) (ctrl.Result, error) {
logger := getLogger(ctx)
now := metav1.Now()
@@ -32,12 +32,12 @@ const ConditionReasonRestoreInProgress = "RestoreInProgress"
//
// A backup must not start against an application a restore is rewriting. It
// would capture a half-restored state at best, and on the CNPG path it does
// worse: every BackupJob reconcile re-applies the Cluster's barman plugin, and
// one landing between the restore re-rendering the Cluster with its new
// serverName and the plugin being attached can put the restored cluster back
// on the source's WAL prefix, where its recovery fails with "Expected empty
// archive". Plan-created runs are the ones that hit this, since nothing else
// looks at restores before creating a BackupJob.
// worse: a BackupJob applies the Cluster's barman plugin before it creates
// its Backup, and an apply landing between the restore re-rendering the
// Cluster with its new serverName and the plugin being attached can put the
// restored cluster back on the source's WAL prefix, where its recovery fails
// with "Expected empty archive". Plan-created runs are the ones that hit
// this, since nothing else looks at restores before creating a BackupJob.
func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) {
app = NormalizeApplicationRef(app)
list := &backupsv1alpha1.RestoreJobList{}
@@ -176,7 +176,9 @@ const (
cnpgRecoveryMaxInspectPods = 5
// Cap on the wall-clock time a BackupJob waits for the barman-cloud
// plugin rollout before the cnpg.io/Backup exists. A Backup that exists
// plugin rollout, counted from the attach, or retries writing the WAL
// archive destination, counted from the first failure since the job last
// left the queue, before the cnpg.io/Backup exists. A Backup that exists
// is not capped: CloudNativePG keeps a healthy one pending while another
// backup of the same Cluster runs, or while it restarts one on another
// instance, so no fixed deadline tells those apart from a stuck one.
@@ -195,6 +197,12 @@ const (
// instance pods must postdate (see cnpgPluginRolloutPending).
backupCondPluginRolledOut = "PluginRolledOut"
// Condition Type recorded while writing the WAL archive destination keeps
// failing. Its LastTransitionTime marks the first failure, so the retry
// budget is not spent by time the job waited for another backup of the
// app before it tried.
backupCondDestinationApplied = "DestinationApplied"
cnpgPodRoleLabel = "cnpg.io/podRole"
cnpgPodRoleInstance = "instance"
)
@@ -321,7 +329,23 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
awaitingRollout := existingBackup == nil && rollout != nil && rollout.Status == metav1.ConditionFalse
attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout)
// The ObjectStore is the Cluster's WAL archive destination, shared by
// every backup of the app, so it is written only before this job's
// Backup exists and only while no other job's Backup is in flight.
writeDestination := existingBackup == nil
if writeDestination {
holder, err := r.otherCNPGBackupInFlight(ctx, j, clusterName)
if err != nil {
return ctrl.Result{}, err
}
if holder != "" {
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf(
"waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder))
}
}
attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination)
if err != nil {
if apierrors.IsNotFound(err) {
// HelmRelease has not yet rendered the Cluster (fresh app, or
@@ -340,7 +364,33 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
}
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to attach barman-cloud plugin to Cluster: %v", err))
if !writeDestination {
// Only the live Cluster was read for a Backup already running;
// a failed read says nothing about that Backup.
return ctrl.Result{}, fmt.Errorf("read cnpg.io/Cluster %s/%s: %w", j.Namespace, clusterName, err)
}
message := fmt.Sprintf("failed to attach barman-cloud plugin to Cluster: %v", err)
if apierrors.IsInvalid(err) || apierrors.IsBadRequest(err) || apierrors.IsForbidden(err) {
return r.markBackupJobFailed(ctx, j, message)
}
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: backupCondDestinationApplied,
Status: metav1.ConditionFalse,
Reason: "ClusterPluginApplyRetrying",
Message: message,
})
failingSince := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied).LastTransitionTime
if cnpgBackupDeadlineExceeded(&failingSince) {
return r.markBackupJobFailed(ctx, j, message)
}
return r.waitCNPGBackupJob(ctx, j, "ClusterPluginApplyRetrying", message)
}
if writeDestination {
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil &&
(ready.Reason == "WaitingForOtherBackup" || ready.Reason == "ClusterPluginApplyRetrying") {
apimeta.RemoveStatusCondition(&j.Status.Conditions, "Ready")
}
}
if attachment.serverName != serverName {
logger.Info("preserving the live Cluster's barman serverName over the strategy template's",
@@ -377,7 +427,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
return ctrl.Result{}, err
}
if pending != "" {
if cnpgBackupDeadlineExceeded(j.Status.StartedAt) {
if cnpgBackupDeadlineExceeded(&rollout.LastTransitionTime) {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
"barman-cloud plugin rollout did not finish within %s: %s", cnpgDefaultBackupDeadline, pending))
}
@@ -486,16 +536,16 @@ func cnpgBackupFailedMessage(b *cnpgtypes.Backup, errText string) string {
return message
}
// cnpgBackupDeadlineExceeded reports whether enough wall-clock time has
// elapsed since the BackupJob started that we should stop waiting for the
// barman-cloud plugin rollout and fail the run. Returns false when
// StartedAt is nil so the very first reconcile (which sets StartedAt) does
// not trip the gate.
func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool {
if startedAt == nil {
// cnpgBackupDeadlineExceeded reports whether more than the backup deadline
// has passed since the BackupJob began the step it is waiting on: the plugin
// rollout, counted from the attach, or retrying the destination write,
// counted from the first failure since the job last left the queue behind
// another backup. Returns false for a nil time.
func cnpgBackupDeadlineExceeded(since *metav1.Time) bool {
if since == nil {
return false
}
return time.Since(startedAt.Time) > cnpgDefaultBackupDeadline
return time.Since(since.Time) > cnpgDefaultBackupDeadline
}
// cnpgPurgeNeeded decides whether the controller should delete the live
@@ -601,7 +651,10 @@ var cnpgClusterGVR = schema.GroupVersionResource{
// With attach unset, a Cluster that has no barman-cloud plugin yet is left
// untouched and only reported missing, so the caller can record that it is
// about to roll the Cluster before it does.
func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach bool) (cnpgPluginAttachment, error) {
//
// With write unset nothing is written: only what the live Cluster carries is
// reported.
func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach, write bool) (cnpgPluginAttachment, error) {
// Live read: the cached Cluster can predate a restore re-render, and the
// serverName, the flavor and the UID below must come from the current
// object.
@@ -626,6 +679,9 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name
return attachment, nil
}
}
if !write {
return attachment, nil
}
serverName = attachment.serverName
// The ObjectStore is named after the Cluster (distinct kind, same
@@ -801,6 +857,60 @@ func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backu
return &list.Items[0], nil
}
// otherCNPGBackupInFlight returns the name of another BackupJob that is still
// running and whose cnpg.io/Backup of clusterName has not finished, or "" when
// there is none. A Backup left behind by a finished or deleted job does not
// hold the queue.
func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) {
list := &cnpgtypes.BackupList{}
if err := r.List(ctx, list, client.InNamespace(j.Namespace)); err != nil {
return "", err
}
var holders []string
for i := range list.Items {
b := &list.Items[i]
owner := b.Labels[backupsv1alpha1.OwningJobNameLabel]
if b.Spec.Cluster.Name != clusterName || owner == "" || owner == j.Name {
continue
}
if b.Status.Phase == cnpgBackupPhaseComplete || b.Status.Phase == cnpgBackupPhaseFailed {
continue
}
ownerJob := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: owner}, ownerJob); err != nil {
if apierrors.IsNotFound(err) {
continue
}
return "", err
}
if ownerJob.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || ownerJob.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
continue
}
holders = append(holders, owner)
}
if len(holders) == 0 {
return "", nil
}
sort.Strings(holders)
return holders[0], nil
}
// waitCNPGBackupJob requeues the BackupJob and records why it waits in its
// Ready condition, writing the status only when the conditions changed.
func (r *BackupJobReconciler) waitCNPGBackupJob(ctx context.Context, j *backupsv1alpha1.BackupJob, reason, message string) (ctrl.Result, error) {
if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
}) {
if err := r.Status().Update(ctx, j); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
}
// createCNPGBackupArtifact materialises a Cozystack Backup resource carrying
// the metadata callers need to drive a future restore. The source app's
// spec.databases / spec.users are snapshotted into Status.UnderlyingResources
@@ -3,6 +3,7 @@ package backupcontroller
import (
"context"
"errors"
"fmt"
"strings"
"testing"
@@ -19,6 +20,7 @@ import (
"k8s.io/client-go/dynamic"
dynamicfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/scheme"
clienttesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -1099,7 +1101,7 @@ func TestApplyClusterPluginBackup_NotFoundOnMissingCluster(t *testing.T) {
},
}
_, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true)
_, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true, true)
if err == nil {
t.Fatalf("expected NotFound error, got nil")
}
@@ -1739,7 +1741,7 @@ func TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache(t *testing.T) {
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"},
}
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true)
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -1949,7 +1951,7 @@ func TestApplyClusterPluginBackup_PreservesLiveServerName(t *testing.T) {
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"},
}
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true)
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -2118,7 +2120,7 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) {
},
}
if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true); err != nil {
if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true, true); err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -2480,6 +2482,434 @@ func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) {
}
}
// newAttachedObjectStoreFixture is a Cluster that already runs the barman-cloud
// plugin, whose ObjectStore currently points at liveDestination; the strategy
// of the fixture's BackupJob points at s3://bucket/.
func newAttachedObjectStoreFixture(t *testing.T, startedAt time.Time, liveDestination string) *cnpgRolloutFixture {
t.Helper()
f := newCNPGRolloutFixture(t, newRolloutBackupJob(startedAt),
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
newRolloutInstancePod(time.Now().Add(-time.Hour)))
store := &cnpgtypes.ObjectStore{
ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster},
Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: liveDestination}},
}
if err := f.c.Create(context.Background(), store); err != nil {
t.Fatalf("seed ObjectStore: %v", err)
}
return f
}
func (f *cnpgRolloutFixture) liveDestination() string {
f.t.Helper()
store := &cnpgtypes.ObjectStore{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil {
f.t.Fatalf("get ObjectStore: %v", err)
}
return store.Spec.Configuration.DestinationPath
}
// seedCNPGBackup creates a cnpg.io/Backup of cluster owned by the BackupJob
// named owner; an empty owner leaves it unlabelled, as a Backup created by
// hand.
func (f *cnpgRolloutFixture) seedCNPGBackup(name, owner, cluster, phase string) {
f.t.Helper()
var labels map[string]string
if owner != "" {
labels = map[string]string{
backupsv1alpha1.OwningJobNameLabel: owner,
backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS,
}
}
b := &cnpgtypes.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: name, Labels: labels},
Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: cluster}},
Status: cnpgtypes.BackupStatus{Phase: phase},
}
if err := f.c.Create(context.Background(), b); err != nil {
f.t.Fatalf("seed cnpg.io/Backup %s: %v", name, err)
}
}
func (f *cnpgRolloutFixture) setJobCondition(cond metav1.Condition) {
f.t.Helper()
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
f.t.Fatalf("get BackupJob: %v", err)
}
j.Status.Conditions = append(j.Status.Conditions, cond)
if err := f.c.Status().Update(context.Background(), j); err != nil {
f.t.Fatalf("seed BackupJob condition: %v", err)
}
}
func (f *cnpgRolloutFixture) seedOtherBackupJob(name string, phase backupsv1alpha1.BackupJobPhase) {
f.t.Helper()
other := newRolloutBackupJob(time.Now().Add(-time.Hour))
other.Name = name
other.Status.Phase = phase
if err := f.c.Create(context.Background(), other); err != nil {
f.t.Fatalf("seed BackupJob %s: %v", name, err)
}
}
// The ObjectStore is the Cluster's WAL archive destination, so a BackupJob
// whose cnpg.io/Backup is already running must not keep re-applying it: a
// second BackupJob of the same app with another destination would otherwise
// flip it back and forth under both backups.
func TestReconcileCNPG_RunningBackupDoesNotReapplyObjectStore(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
j, _ := f.pass()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("BackupJob failed: %s", j.Status.Message)
}
if got := f.liveDestination(); got != "s3://other/" {
t.Fatalf("a pass over a running backup re-applied the ObjectStore: destination %q", got)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" {
t.Fatalf("a running backup waits on itself: %+v", ready)
}
}
func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
cases := []struct {
name string
otherJob backupsv1alpha1.BackupJobPhase
seedJob bool
otherBackup string
owner string
cluster string
wantWait bool
}{
{name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true},
{name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true},
{name: "a completed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseComplete},
{name: "a failed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseFailed},
{name: "a Backup of a failed job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "pending"},
{name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"},
{name: "a Backup whose job is gone does not", otherBackup: "pending"},
{name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"},
{name: "a Backup created by hand does not", otherBackup: "running", owner: "-"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
if tc.seedJob {
f.seedOtherBackupJob("bj-other", tc.otherJob)
}
owner, cluster := "bj-other", rolloutCluster
if tc.owner == "-" {
owner = ""
}
if tc.cluster != "" {
cluster = tc.cluster
}
f.seedCNPGBackup("bj-other-cnpg", owner, cluster, tc.otherBackup)
// client-go refuses a Get with an empty name before it reaches
// the apiserver; the fake client answers NotFound instead.
statusWrites := 0
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if key.Name == "" {
return errors.New("resource name may not be empty")
}
return cl.Get(ctx, key, obj, opts...)
},
SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
statusWrites++
return cl.SubResource(sub).Update(ctx, obj, opts...)
},
})
j, backups := f.pass()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("BackupJob failed: %s", j.Status.Message)
}
ours := 0
for _, b := range backups {
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
ours++
}
}
ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready")
if tc.wantWait {
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("the waiting job applied its destination over a running backup: %q", got)
}
if ours != 0 {
t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours)
}
if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") {
t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready)
}
statusWrites = 0
f.pass()
if statusWrites != 0 {
t.Errorf("a second pass with the same holder wrote the BackupJob status %d time(s)", statusWrites)
}
held := &cnpgtypes.Backup{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil {
t.Fatalf("get holder Backup: %v", err)
}
held.Status.Phase = cnpgBackupPhaseComplete
if err := f.c.Update(context.Background(), held); err != nil {
t.Fatalf("complete holder Backup: %v", err)
}
j, _ = f.pass()
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" {
t.Errorf("Ready still names the finished holder: %+v", ready)
}
return
}
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, want the job's own s3://bucket/", got)
}
if ours != 1 {
t.Errorf("got %d cnpg.io/Backups for the job, want 1", ours)
}
})
}
}
func TestReconcileCNPG_ObjectStoreApplyErrors(t *testing.T) {
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
webhookDown := apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io": connect: connection refused`))
invalid := apierrors.NewInvalid(schema.GroupKind{Group: cnpgtypes.BarmanGroupVersion.Group, Kind: "ObjectStore"}, rolloutCluster, nil)
cases := []struct {
name string
err error
startedAt time.Time
// failingSince seeds an earlier failure of the same write.
failingSince time.Time
wantPhase backupsv1alpha1.BackupJobPhase
}{
{name: "a transient error is retried", err: webhookDown, startedAt: time.Now()},
{name: "a transient error failing since past the deadline fails", err: webhookDown, startedAt: expired, failingSince: expired, wantPhase: backupsv1alpha1.BackupJobPhaseFailed},
{name: "a transient error after a long wait is retried", err: webhookDown, startedAt: expired},
{name: "an invalid object fails at once", err: invalid, startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed},
{name: "a bad request fails at once", err: apierrors.NewBadRequest("malformed"), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed},
{name: "a forbidden write fails at once", err: apierrors.NewForbidden(schema.GroupResource{Resource: "objectstores"}, rolloutCluster, errors.New("denied")), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, tc.startedAt, "s3://other/")
if !tc.failingSince.IsZero() {
f.setJobCondition(metav1.Condition{
Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying",
Message: "earlier failure", LastTransitionTime: metav1.NewTime(tc.failingSince),
})
}
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if _, ok := obj.(*cnpgtypes.ObjectStore); ok {
return tc.err
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
j, backups := f.pass()
if j.Status.Phase != tc.wantPhase {
t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase)
}
if len(backups) != 0 {
t.Errorf("a cnpg.io/Backup was created although the ObjectStore was not applied: %+v", backups)
}
if tc.wantPhase == "" {
ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready")
if ready == nil || ready.Reason != "ClusterPluginApplyRetrying" || !strings.Contains(ready.Message, "connection refused") {
t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying carrying the error", ready)
}
}
})
}
}
// A write that recovers clears the retry state, so a later failure gets a
// fresh budget and Ready no longer reports the retry.
func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
failing := true
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if _, ok := obj.(*cnpgtypes.ObjectStore); ok && failing {
return apierrors.NewInternalError(errors.New("webhook down"))
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
j, _ := f.pass()
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond == nil || cond.Status != metav1.ConditionFalse {
t.Fatalf("%s = %+v after a failed write, want False", backupCondDestinationApplied, cond)
}
failing = false
j, backups := f.pass()
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups after the write recovered, want 1", len(backups))
}
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil {
t.Errorf("%s = %+v left over after the write recovered", backupCondDestinationApplied, cond)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "ClusterPluginApplyRetrying" {
t.Errorf("Ready still reports the retry: %+v", ready)
}
}
// An error reading the holder's BackupJob is not taken as "no holder".
func TestReconcileCNPG_HolderLookupErrorIsReturned(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if key.Name == "bj-other" {
return apierrors.NewServiceUnavailable("cache not synced")
}
return cl.Get(ctx, key, obj, opts...)
},
})
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
t.Fatalf("reconcileCNPG swallowed the holder lookup error")
}
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination written although the holder could not be read: %q", got)
}
}
// The rollout deadline counts from when the plugin was attached, not from
// the job's start, which may lie behind a long wait for another backup.
func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) {
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired),
newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour)))
f.pass() // latch
j, _ := f.pass()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("a rollout that just began failed on the job's old start: %s", j.Status.Message)
}
if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" {
t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond)
}
rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
rollout.LastTransitionTime = metav1.NewTime(expired)
if err := f.c.Status().Update(context.Background(), j); err != nil {
t.Fatalf("age the rollout: %v", err)
}
j, _ = f.pass()
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") {
t.Fatalf("phase = %q (%s), want Failed on a rollout older than the deadline", j.Status.Phase, j.Status.Message)
}
}
// Waiting for another backup resets the retry budget of the destination
// write, so the queue does not spend it.
func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) {
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
f := newAttachedObjectStoreFixture(t, expired, "s3://other/")
f.setJobCondition(metav1.Condition{
Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying",
Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired),
})
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
j, _ := f.pass()
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil {
t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond)
}
held := &cnpgtypes.Backup{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil {
t.Fatalf("get holder Backup: %v", err)
}
held.Status.Phase = cnpgBackupPhaseComplete
if err := f.c.Update(context.Background(), held); err != nil {
t.Fatalf("complete holder Backup: %v", err)
}
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if _, ok := obj.(*cnpgtypes.ObjectStore); ok {
return apierrors.NewInternalError(errors.New("webhook down"))
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
j, _ = f.pass()
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("the first failure after the queue failed the job: %s", j.Status.Message)
}
}
func TestOtherCNPGBackupInFlight_IgnoresTheJobsOwnBackup(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
holder, err := f.r.otherCNPGBackupInFlight(context.Background(), j, rolloutCluster)
if err != nil {
t.Fatalf("otherCNPGBackupInFlight: %v", err)
}
if holder != "" {
t.Fatalf("the job waits on its own Backup: holder %q", holder)
}
}
// Once the job's Backup exists the Cluster is only read; a failed read is
// returned for a retry and neither fails the running backup nor starts the
// destination retry budget.
func TestReconcileCNPG_ClusterReadErrorDoesNotFailRunningBackup(t *testing.T) {
for name, readErr := range map[string]error{
"internal error": apierrors.NewInternalError(errors.New("blip")),
// A write answered Forbidden fails the job; a read must not.
"forbidden": apierrors.NewForbidden(schema.GroupResource{Group: cnpgtypes.GroupName, Resource: "clusters"}, rolloutCluster, errors.New("denied")),
} {
t.Run(name, func(t *testing.T) { testClusterReadErrorKeepsBackup(t, readErr) })
}
}
func testClusterReadErrorKeepsBackup(t *testing.T, readErr error) {
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
f := newAttachedObjectStoreFixture(t, expired, "s3://other/")
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
f.setJobCondition(metav1.Condition{
Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying",
Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired),
})
dyn := f.r.Interface.(*dynamicfake.FakeDynamicClient)
dyn.PrependReactor("get", "clusters", func(clienttesting.Action) (bool, runtime.Object, error) {
return true, nil, readErr
})
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
t.Fatalf("a failed Cluster read was not returned for a retry")
}
persisted := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("a failed Cluster read failed the running backup: %s", persisted.Status.Message)
}
}
// cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose
// Cluster and instance pods a test swaps between passes, standing in for
// CNPG rolling the instances.