fix(postgres): hold a backup until the plugin sees its new destination

The plugin-barman-cloud sidecar reads the ObjectStore through its own
cache, kept for ten seconds (DefaultTTLSeconds in plugin-barman-cloud
v0.13.0) and invalidated only by the sidecar's own writes, and WAL
archiving keeps it warm. The pass that changed the destination created
the cnpg.io/Backup a moment later, so the base backup could be taken
against the previous destination while the artifact names the new one,
and a restore from it finds nothing. On a live cluster with the shipped
plugin, a Backup created two to four seconds after switching the
destination landed in the old bucket in two of three tries; with a
fifteen second wait it landed in the new one every time.

The apply that changes the ObjectStore configuration now stamps the
time on the ObjectStore itself, so the record cannot be lost apart from
the change, and the job creates its Backup only once that change is
older than the cache can be. A change is judged on the configuration a
server-side dry run of the same apply returns, so values the apiserver
defaults or another field manager owns do not read as a change on every
pass.

That wait leaves a job with the destination written and no Backup yet,
which the queue, looking only for Backups in flight, did not see: a
second job with another destination wrote its own, the first took that
as a change and stamped again, and the two restarted each other's wait
forever. Every apply of the ObjectStore therefore also names the job
that made it, which only a job without a Backup of its own and let
through by the queue does, and the queue holds every other job while
that one runs and has no Backup yet. Once it has one, that Backup holds
the destination for as long as CloudNativePG runs it, as before, so a
job stuck after its Backup finished does not hold every later backup of
the application. This also covers a job waiting for the
rollout of the plugin it attached, during which a second job used to
switch the destination as well, including when its own write left the
configuration as a restore had re-rendered it.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-12 15:13:58 +03:00
1 parent 59a11c1788
commit 1d22013646
2 files changed
+726 -29

No files matched your search

@@ -15,6 +15,7 @@ import (
"time"
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
@@ -233,9 +234,30 @@ const (
// cannot be lost to a failed BackupJob status write after the attach.
cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at"
// How far ahead of this controller's clock an attach record may be
// before it is taken as corrupt rather than as written by a leader whose
// clock ran ahead.
// Annotation on the ObjectStore carrying, as RFC 3339, when a backup job
// last changed its configuration. It is written by the apply that makes
// the change, so it cannot be lost apart from it.
cnpgDestinationChangedAtAnnotation = "backups.cozystack.io/destination-changed-at"
// Annotation on the ObjectStore naming the BackupJob whose apply last
// wrote it. Written by that apply, it holds the destination for that job
// while it runs and has no cnpg.io/Backup of its own yet; from then on
// the queue sees that Backup instead.
cnpgDestinationWrittenByAnnotation = "backups.cozystack.io/destination-written-by"
// How long after an ObjectStore configuration change a Backup is held.
// The plugin-barman-cloud sidecar reads the ObjectStore through its own
// cache, kept for DefaultTTLSeconds = 10 (plugin-barman-cloud v0.13.0,
// internal/cnpgi/instance/internal/client/client.go) and invalidated
// only by the sidecar's own writes, so a Backup taken sooner can use the
// previous configuration and land in the previous bucket. The margin
// covers the cache's whole-second expiry and the record's whole-second
// timestamp.
cnpgObjectStoreCacheSettle = 15 * time.Second
// How far ahead of this controller's clock a recorded time, of a plugin
// attach or of a destination change, may be before it is taken as
// corrupt rather than as written by a leader whose clock ran ahead.
cnpgAttachClockSkewTolerance = 5 * time.Minute
cnpgPodRoleLabel = "cnpg.io/podRole"
@@ -366,7 +388,9 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
// The ObjectStore is the Cluster's WAL archive destination, shared by
// every backup of the app, so it is written only before this job's
// Backup exists and only while no other job's Backup is in flight.
// Backup exists, and only while no other job's Backup is in flight and
// no other running job without a Backup yet holds the destination it
// last wrote.
writeDestination := existingBackup == nil
if writeDestination {
// The gate in Reconcile sees only a job that has not started, and
@@ -386,6 +410,11 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
if err != nil {
return ctrl.Result{}, err
}
if holder == "" {
if holder, err = r.destinationClaimHolder(ctx, j, clusterName); err != nil {
return ctrl.Result{}, err
}
}
if holder != "" {
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForOtherBackup"))
@@ -403,7 +432,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
attemptedAt := metav1.Now()
attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination)
attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination, j.Name)
if err != nil {
if apierrors.IsNotFound(err) {
// HelmRelease has not yet rendered the Cluster (fresh app, or
@@ -504,7 +533,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
// The Cluster's record covers an attach whose status write was lost.
attachedAt := rollout.LastTransitionTime
recorded, ok, err := cnpgRecordedAttach(attachment.attachedAt)
recorded, ok, err := cnpgRecordedTime(attachment.attachedAt)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
"cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v",
@@ -548,13 +577,11 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
apimeta.RemoveStatusCondition(&j.Status.Conditions, "Ready")
rolledOut = true
} else if existingBackup == nil && attachment.attachedAt != "" {
// Another backup job attached the plugin. Its rollout can still be
// running while the attach is recent; past the rollout deadline
// this job stops waiting, as the attaching job normally has judged
// the rollout by then unless a wait of its own moved its deadline
// on. The record is that job's:
// one this job cannot use does not fail it.
recorded, _, err := cnpgRecordedAttach(attachment.attachedAt)
// Another backup job attached the plugin, and its rollout can still
// be running while the attach is recent; past the rollout deadline
// this job stops waiting. The record is that job's: one this job
// cannot use does not fail it.
recorded, _, err := cnpgRecordedTime(attachment.attachedAt)
if err != nil {
logger.Info("ignoring an unusable attach record another backup job left",
"cluster", clusterName, "annotation", cnpgPluginAttachedAtAnnotation, "error", err)
@@ -577,6 +604,12 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
"cluster", clusterName, "live", attachment.flavor, "application", specFlavor)
}
if changedAt := attachment.destinationChangedAt; !changedAt.IsZero() && time.Since(changedAt) < cnpgObjectStoreCacheSettle {
return r.waitCNPGBackupJob(ctx, j, "WaitingForDestinationToSettle", fmt.Sprintf(
"the WAL archive destination of cnpg.io/Cluster %s/%s changed at %s; the barman-cloud plugin can read the previous one for up to %s",
j.Namespace, clusterName, changedAt.UTC().Format(time.RFC3339), cnpgObjectStoreCacheSettle))
}
written := cnpgDestinationOf(rendered)
written.ServerName = serverName
cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, existingBackup, written)
@@ -780,7 +813,7 @@ var cnpgClusterGVR = schema.GroupVersionResource{
//
// With write unset nothing is written: only what the live Cluster carries is
// reported.
func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach, write bool) (cnpgPluginAttachment, error) {
func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach, write bool, jobName string) (cnpgPluginAttachment, error) {
// Live read: the cached Cluster can predate a restore re-render, and the
// serverName, the flavor and the UID below must come from the current
// object.
@@ -836,9 +869,21 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name
RetentionPolicy: t.BarmanObjectStore.RetentionPolicy,
InstanceSidecarConfiguration: barmanSidecarConfiguration(),
}
changedAt, err := r.objectStoreChangeRecord(ctx, objStore)
if err != nil {
return cnpgPluginAttachment{}, err
}
// Only a job without a Backup of its own writes, and the queue let it
// through, so it holds the destination from this write while it runs
// and has not created its Backup.
objStore.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: jobName}
if !changedAt.IsZero() {
objStore.Annotations[cnpgDestinationChangedAtAnnotation] = changedAt.UTC().Format(time.RFC3339)
}
if err := r.Patch(ctx, objStore, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil {
return cnpgPluginAttachment{}, fmt.Errorf("apply ObjectStore %s/%s: %w", namespace, objStoreName, err)
}
attachment.destinationChangedAt = changedAt
patch := newCNPGClusterPatch(namespace, clusterName)
patch.Spec.Plugins = []cnpgtypes.PluginConfiguration{buildBarmanPlugin(objStoreName, serverName)}
@@ -857,6 +902,83 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name
return attachment, nil
}
// objectStoreChangeRecord returns the cnpgDestinationChangedAtAnnotation the
// apply of desired should carry: the time of this write when it changes the
// configuration, otherwise the record the live ObjectStore already carries,
// re-sent so the apply does not delete it, and zero when there is none.
// "Changes" is judged on the configuration a server-side dry run of the same
// apply returns, not on desired, so values the apiserver defaults or another
// field manager owns do not read as a change on every pass. Creating the
// ObjectStore stamps no time: the plugin cannot hold a stale copy of an
// object that did not exist, short of one deleted and recreated by hand
// within the cache's lifetime.
func (r *BackupJobReconciler) objectStoreChangeRecord(ctx context.Context, desired *cnpgtypes.ObjectStore) (time.Time, error) {
live := &cnpgtypes.ObjectStore{}
if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(desired), live); err != nil {
if apierrors.IsNotFound(err) {
return time.Time{}, nil
}
return time.Time{}, fmt.Errorf("read ObjectStore %s/%s: %w", desired.Namespace, desired.Name, err)
}
dry := desired.DeepCopy()
if err := r.Patch(ctx, dry, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership, client.DryRunAll); err != nil {
return time.Time{}, fmt.Errorf("dry-run apply ObjectStore %s/%s: %w", desired.Namespace, desired.Name, err)
}
if !apiequality.Semantic.DeepEqual(live.Spec.Configuration, dry.Spec.Configuration) {
return time.Now(), nil
}
recorded, _, err := cnpgRecordedTime(live.Annotations[cnpgDestinationChangedAtAnnotation])
if err != nil {
// Nothing says when the cache expires; this write's time is the
// latest it can be.
return time.Now(), nil
}
return recorded, nil
}
// destinationClaimHolder returns the BackupJob other than j that the
// ObjectStore of clusterName names as the last to write it, while that job
// is still running and has no cnpg.io/Backup of its own, or "" when there is
// none. Such a job wrote the destination before otherCNPGBackupInFlight can
// see it: it can still be waiting for the plugin cache, or for the rollout of
// the plugin it attached. Once it has a Backup, that Backup holds the
// destination while CloudNativePG runs it, and the claim stops holding, so a
// job stuck after its Backup finished does not hold every later one.
func (r *BackupJobReconciler) destinationClaimHolder(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) {
store := &cnpgtypes.ObjectStore{}
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: clusterName}, store); err != nil {
if apierrors.IsNotFound(err) {
return "", nil
}
return "", fmt.Errorf("read ObjectStore %s/%s: %w", j.Namespace, clusterName, err)
}
holder := store.Annotations[cnpgDestinationWrittenByAnnotation]
if holder == "" || holder == j.Name {
return "", nil
}
holderJob := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: holder}, holderJob); err != nil {
if apierrors.IsNotFound(err) {
return "", nil
}
return "", err
}
if holderJob.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || holderJob.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
return "", nil
}
holderBackups := &cnpgtypes.BackupList{}
if err := r.apiReader().List(ctx, holderBackups, client.InNamespace(j.Namespace), client.MatchingLabels{
backupsv1alpha1.OwningJobNameLabel: holder,
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
}); err != nil {
return "", fmt.Errorf("list cnpg.io/Backups of BackupJob %s/%s: %w", j.Namespace, holder, err)
}
if len(holderBackups.Items) > 0 {
return "", nil
}
return holder, nil
}
// cnpgPluginAttachment is what applyClusterPluginBackup read off the live
// Cluster before applying the plugin.
type cnpgPluginAttachment struct {
@@ -869,8 +991,11 @@ type cnpgPluginAttachment struct {
// attachedAt is the cnpgPluginAttachedAtAnnotation the Cluster carried,
// "" when it carries no record: the plugin was attached by the chart, or
// before the record existed.
attachedAt string
clusterStatus cnpgtypes.ClusterStatus
attachedAt string
// destinationChangedAt is the cnpgDestinationChangedAtAnnotation this
// pass applied to the ObjectStore, zero when it wrote none.
destinationChangedAt time.Time
clusterStatus cnpgtypes.ClusterStatus
}
// cnpgPluginRolloutPending returns why the rollout that attaching the
@@ -916,12 +1041,13 @@ func (r *BackupJobReconciler) cnpgPluginRolloutPending(ctx context.Context, name
return "", nil
}
// cnpgRecordedAttach parses a cnpgPluginAttachedAtAnnotation value; ok is
// false for "", a Cluster that carries no record. A record further ahead of
// this controller's clock than cnpgAttachClockSkewTolerance is an error:
// every instance would predate it, and a deadline counted from it would not
// run out for as long as it lies ahead.
func cnpgRecordedAttach(raw string) (recorded time.Time, ok bool, err error) {
// cnpgRecordedTime parses a recorded time annotation, the plugin attach
// record on the Cluster or the destination change record on the ObjectStore;
// ok is false for "", an object that carries no record. A record further
// ahead of this controller's clock than cnpgAttachClockSkewTolerance is an
// error: every instance would predate it, and a deadline counted from it
// would not run out for as long as it lies ahead.
func cnpgRecordedTime(raw string) (recorded time.Time, ok bool, err error) {
if raw == "" {
return time.Time{}, false, nil
}
@@ -1103,7 +1103,7 @@ func TestApplyClusterPluginBackup_NotFoundOnMissingCluster(t *testing.T) {
},
}
_, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true, true)
_, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true, true, "bj")
if err == nil {
t.Fatalf("expected NotFound error, got nil")
}
@@ -1743,7 +1743,7 @@ func TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache(t *testing.T) {
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"},
}
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true)
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -1953,7 +1953,7 @@ func TestApplyClusterPluginBackup_PreservesLiveServerName(t *testing.T) {
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"},
}
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true)
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -2122,7 +2122,7 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) {
},
}
if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true, true); err != nil {
if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true, true, "bj"); err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -2736,6 +2736,17 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, want the job's own s3://bucket/", got)
}
if ready != nil && ready.Reason == "WaitingForOtherBackup" {
t.Errorf("Ready = %+v, the job was held", ready)
}
f.settleDestination()
_, backups = f.pass()
ours = 0
for _, b := range backups {
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
ours++
}
}
if ours != 1 {
t.Errorf("got %d cnpg.io/Backups for the job, want 1", ours)
}
@@ -2819,6 +2830,8 @@ func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) {
}
failing = false
j, _ = f.pass()
f.settleDestination()
j, backups := f.pass()
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups after the write recovered, want 1", len(backups))
@@ -3121,6 +3134,562 @@ func TestReconcileCNPG_AttachRecordOlderThanTheLatchKeepsTheLatch(t *testing.T)
}
}
func (f *cnpgRolloutFixture) liveObjectStore() *cnpgtypes.ObjectStore {
f.t.Helper()
store := &cnpgtypes.ObjectStore{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil {
f.t.Fatalf("get ObjectStore: %v", err)
}
return store
}
// setDestinationStamp rewrites the destination-change record on the live
// ObjectStore, as a write that long ago, or a corrupt one, would have left it.
func (f *cnpgRolloutFixture) setDestinationStamp(value string) {
f.t.Helper()
store := f.liveObjectStore()
if store.Annotations == nil {
store.Annotations = map[string]string{}
}
store.Annotations[cnpgDestinationChangedAtAnnotation] = value
if err := f.c.Update(context.Background(), store); err != nil {
f.t.Fatalf("stamp ObjectStore: %v", err)
}
}
// settleDestination ages the destination-change record, when the ObjectStore
// carries one, as if the plugin's cache had expired since that write.
func (f *cnpgRolloutFixture) settleDestination() {
f.t.Helper()
if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok {
f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339))
}
}
// addJobWithDestination adds BackupJob name of the same application, resolved
// to a strategy of its own that writes destination, and returns a function
// running one pass of it.
func (f *cnpgRolloutFixture) addJobWithDestination(name, destination string) func() *backupsv1alpha1.BackupJob {
f.t.Helper()
ctx := context.Background()
strategy := &strategyv1alpha1.CNPG{
ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy-" + name},
Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: destination},
}},
}
if err := f.c.Create(ctx, strategy); err != nil {
f.t.Fatalf("seed strategy: %v", err)
}
job := newRolloutBackupJob(time.Now().Add(-time.Hour))
job.Name = name
if err := f.c.Create(ctx, job); err != nil {
f.t.Fatalf("seed BackupJob %s: %v", name, err)
}
group := strategyv1alpha1.GroupVersion.Group
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &group, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}}
return func() *backupsv1alpha1.BackupJob {
f.t.Helper()
key := client.ObjectKey{Namespace: rolloutNS, Name: name}
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(ctx, key, j); err != nil {
f.t.Fatalf("get BackupJob %s: %v", name, err)
}
if _, err := f.r.reconcileCNPG(ctx, j, resolved); err != nil {
f.t.Fatalf("reconcileCNPG %s: %v", name, err)
}
if err := f.c.Get(ctx, key, j); err != nil {
f.t.Fatalf("get BackupJob %s after reconcile: %v", name, err)
}
return j
}
}
// Two jobs of the application with different destinations, neither with a
// Backup yet, would otherwise take turns rewriting the ObjectStore, each
// write restarting the other's wait for the plugin cache so that neither
// ever creates one. The job whose write came first holds the destination,
// by its claim until it has a Backup and by that Backup afterwards, and the
// other waits for it.
func TestReconcileCNPG_JobsWithDifferentDestinationsTakeTurns(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://third/")
pass2 := f.addJobWithDestination("bj2", "s3://other/")
var j2 *backupsv1alpha1.BackupJob
for range 5 {
f.pass()
f.settleDestination()
j2 = pass2()
f.settleDestination()
}
backups := &cnpgtypes.BackupList{}
if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil {
t.Fatalf("list cnpg.io/Backups: %v", err)
}
if len(backups.Items) != 1 || backups.Items[0].Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" {
t.Fatalf("cnpg.io/Backups = %+v, want exactly one, of bj, whose write came first", backups.Items)
}
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, want bj's s3://bucket/ under its running Backup", got)
}
if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") {
t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready)
}
}
// The claim names the job whose write last reached the ObjectStore, whether
// it changed the configuration, created the object or left it as it was; the
// change record is re-sent as it is by a write that changes nothing.
func TestReconcileCNPG_ClaimNamesTheJobThatWroteTheDestination(t *testing.T) {
t.Run("changed", func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
f.pass()
if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" {
t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got)
}
})
t.Run("created", func(t *testing.T) {
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
newRolloutInstancePod(time.Now().Add(-time.Hour)))
f.pass()
store := f.liveObjectStore()
if got := store.Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" {
t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got)
}
if _, ok := store.Annotations[cnpgDestinationChangedAtAnnotation]; ok {
t.Errorf("creating the ObjectStore stamped %s, which would hold the Backup for nothing", cnpgDestinationChangedAtAnnotation)
}
})
for name, recorded := range map[string]map[string]string{
"unchanged": {cnpgDestinationWrittenByAnnotation: "bj-old"},
"unchanged with a stamp": {cnpgDestinationWrittenByAnnotation: "bj-old", cnpgDestinationChangedAtAnnotation: "2026-01-02T03:04:05Z"},
} {
t.Run(name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/")
store := f.liveObjectStore()
store.Annotations = recorded
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed claim: %v", err)
}
var sent []map[string]string
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
po := &client.PatchOptions{}
po.ApplyOptions(opts)
if _, ok := obj.(*cnpgtypes.ObjectStore); ok && !slices.Contains(po.DryRun, metav1.DryRunAll) {
sent = append(sent, obj.GetAnnotations())
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
f.pass()
if len(sent) != 1 {
t.Fatalf("got %d ObjectStore writes, want 1", len(sent))
}
if got := sent[0][cnpgDestinationWrittenByAnnotation]; got != "bj" {
t.Errorf("the write sent %s = %q, want bj, the job that wrote", cnpgDestinationWrittenByAnnotation, got)
}
if got, want := sent[0][cnpgDestinationChangedAtAnnotation], recorded[cnpgDestinationChangedAtAnnotation]; got != want {
t.Errorf("the write sent %s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, want)
}
})
}
}
// A claim stops holding the destination once its job has finished or is
// gone, so the next job writes its own.
func TestReconcileCNPG_ClaimOfAFinishedJobDoesNotHold(t *testing.T) {
for name, phase := range map[string]backupsv1alpha1.BackupJobPhase{
"succeeded": backupsv1alpha1.BackupJobPhaseSucceeded,
"failed": backupsv1alpha1.BackupJobPhaseFailed,
"gone": "",
} {
t.Run(name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
if name != "gone" {
f.seedOtherBackupJob("bj-other", phase)
}
store := f.liveObjectStore()
store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"}
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed claim: %v", err)
}
j, _ := f.pass()
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, want bj's s3://bucket/", got)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" {
t.Errorf("Ready = %+v, held by a %s job", ready, name)
}
})
}
}
// A job that attaches the plugin writes a configuration the ObjectStore may
// already carry, for instance after a restore re-rendered the Cluster
// without the plugin. Its write still claims the destination, so a job with
// another one waits through the rollout instead of switching it.
func TestReconcileCNPG_UnchangedWriteClaimsTheDestinationForTheRollout(t *testing.T) {
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour)))
f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded)
store := &cnpgtypes.ObjectStore{
ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster,
Annotations: map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"}},
Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: "s3://bucket/"}},
}
if err := f.c.Create(context.Background(), store); err != nil {
t.Fatalf("seed ObjectStore: %v", err)
}
pass2 := f.addJobWithDestination("bj2", "s3://other/")
f.pass() // latch
if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForPluginRollout" {
t.Fatalf("Ready = %+v, want bj waiting for the rollout of the plugin it attached", apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
j2 := pass2()
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, switched during bj's rollout", got)
}
if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") {
t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready)
}
}
// The claim covers only the window before its job has a Backup the queue
// can see. Once it has one, the Backup in flight holds the destination, for
// as long as CloudNativePG runs it, and a Backup that finished does not, even
// if its job is stuck before it finishes, so that job cannot hold every later
// backup of the application for as long as it lives.
func TestReconcileCNPG_ClaimOfAJobWithItsOwnBackupDoesNotHold(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, cnpgBackupPhaseComplete)
store := f.liveObjectStore()
store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"}
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed claim: %v", err)
}
j, _ := f.pass()
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, held by a job whose Backup already finished", got)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" {
t.Errorf("Ready = %+v, held by a job whose Backup already finished", ready)
}
}
// A claim whose job cannot be read cannot be taken as released: writing over
// it would let two jobs flip the destination again. The error goes back to
// the controller for a retry.
func TestReconcileCNPG_ClaimHolderLookupErrorIsReturned(t *testing.T) {
unavailable := apierrors.NewServiceUnavailable("apiserver unavailable")
for name, setup := range map[string]func(f *cnpgRolloutFixture){
"holder job read fails": func(f *cnpgRolloutFixture) {
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.BackupJob); ok && key.Name == "bj-other" {
return unavailable
}
return cl.Get(ctx, key, obj, opts...)
},
})
},
"holder Backups list fails": func(f *cnpgRolloutFixture) {
holders := labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj-other", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS}
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
lo := &client.ListOptions{}
lo.ApplyOptions(opts)
if _, ok := list.(*cnpgtypes.BackupList); ok && lo.LabelSelector != nil && lo.LabelSelector.Matches(holders) && !lo.LabelSelector.Matches(labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS}) {
return unavailable
}
return cl.List(ctx, list, opts...)
},
})
},
} {
t.Run(name, func(t *testing.T) {
assertClaimHolderLookupErrorReturned(t, setup)
})
}
}
func assertClaimHolderLookupErrorReturned(t *testing.T, setup func(f *cnpgRolloutFixture)) {
t.Helper()
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
store := f.liveObjectStore()
store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"}
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed claim: %v", err)
}
setup(f)
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
t.Fatalf("reconcileCNPG swallowed the claim holder lookup error")
}
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination written although the claim holder could not be read: %q", got)
}
}
// A running job that holds the claim keeps the destination while it has no
// Backup yet, for instance while the plugin it attached rolls the instances.
func TestReconcileCNPG_ClaimOfARunningJobHolds(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
store := f.liveObjectStore()
store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"}
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed claim: %v", err)
}
j, backups, res := f.passResult()
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination = %q, rewritten under bj-other's claim", got)
}
if len(backups) != 0 {
t.Errorf("a cnpg.io/Backup was created under another job's claim: %+v", backups)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") {
t.Errorf("Ready = %+v, want WaitingForOtherBackup naming bj-other", ready)
}
if res.RequeueAfter == 0 {
t.Errorf("a held job is not requeued: %+v", res)
}
}
// The plugin sidecar reads the ObjectStore through a cache that nothing this
// controller writes invalidates, so a Backup created right after the
// destination changed can be taken against the previous one. The job holds
// the Backup until that cache has expired, and creates it afterwards.
func TestReconcileCNPG_BackupWaitsForThePluginCacheAfterADestinationChange(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
j, backups, res := f.passResult()
if got := f.liveDestination(); got != "s3://bucket/" {
t.Fatalf("destination = %q, want the job's s3://bucket/", got)
}
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created in the pass that changed the destination: %+v", backups)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForDestinationToSettle" {
t.Errorf("Ready = %+v, want WaitingForDestinationToSettle", ready)
}
if res.RequeueAfter == 0 {
t.Errorf("a waiting job is not requeued: %+v", res)
}
stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation])
if err != nil || time.Since(stamp) > time.Minute {
t.Fatalf("%s = %v (%v), want the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err)
}
if _, backups = f.pass(); len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created while the plugin cache could still hold the old destination: %+v", backups)
}
f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339))
if j, backups = f.pass(); len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups once the cache expired, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
}
// A write that leaves the configuration as it was cannot leave a stale
// cache behind: the Backup is created at once, and the record of the last
// change is re-sent as it is, since an apply that left it out would delete it.
func TestReconcileCNPG_UnchangedDestinationDoesNotWait(t *testing.T) {
const stamp = "2026-01-02T03:04:05Z"
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/")
f.setDestinationStamp(stamp)
if j, backups := f.pass(); len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups for an unchanged destination, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
if got := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; got != stamp {
t.Errorf("%s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, stamp)
}
}
// The apiserver answers an apply with the merged object: fields it defaults,
// or that another field manager owns, are there although the job never sets
// them. Judged against what the job sends, every pass would read as a change
// and the Backup would never be created; judged against the dry run of the
// same apply, an unchanged configuration stays unchanged.
func TestReconcileCNPG_FieldsTheJobDoesNotSetAreNotAChange(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/")
store := f.liveObjectStore()
store.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"}
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed ObjectStore: %v", err)
}
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
po := &client.PatchOptions{}
po.ApplyOptions(opts)
if os, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) {
os.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"}
return nil
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
if j, backups := f.pass(); len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups for a configuration only the server filled in, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok {
t.Errorf("%s was set by a write that changed nothing", cnpgDestinationChangedAtAnnotation)
}
}
// Without the live ObjectStore, or the dry run of the apply, the job cannot
// tell whether its write changes the destination, and taking the write as
// unchanged would create the Backup at once against a cache that may still
// hold the previous one. Either failure is retried like a failed write.
func TestReconcileCNPG_UnreadableObjectStoreIsRetried(t *testing.T) {
unavailable := apierrors.NewServiceUnavailable("apiserver unavailable")
for _, tc := range []struct {
name string
setup func(f *cnpgRolloutFixture)
}{
{name: "live read fails", setup: func(f *cnpgRolloutFixture) {
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*cnpgtypes.ObjectStore); ok {
return unavailable
}
return cl.Get(ctx, key, obj, opts...)
},
})
}},
{name: "claim read fails", setup: func(f *cnpgRolloutFixture) {
reads := 0
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*cnpgtypes.ObjectStore); ok {
if reads++; reads == 1 {
return unavailable
}
}
return cl.Get(ctx, key, obj, opts...)
},
})
}},
{name: "live read at the write fails", setup: func(f *cnpgRolloutFixture) {
reads := 0
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*cnpgtypes.ObjectStore); ok {
if reads++; reads > 1 {
return unavailable
}
}
return cl.Get(ctx, key, obj, opts...)
},
})
}},
{name: "dry run fails", setup: func(f *cnpgRolloutFixture) {
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
po := &client.PatchOptions{}
po.ApplyOptions(opts)
if _, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) {
return unavailable
}
return cl.Patch(ctx, obj, patch, opts...)
},
})
}},
} {
t.Run(tc.name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
tc.setup(f)
assertObjectStoreReadRetried(t, f)
})
}
}
// assertObjectStoreReadRetried runs one pass and checks it is retried, either
// by returning the error to the controller or by waiting with
// ClusterPluginApplyRetrying, without a Backup, a destination write or a
// failed job.
func assertObjectStoreReadRetried(t *testing.T, f *cnpgRolloutFixture) {
t.Helper()
ctx := context.Background()
key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"}
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(ctx, key, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
res, reconcileErr := f.r.reconcileCNPG(ctx, j, f.resolved)
if err := f.c.Get(ctx, key, j); err != nil {
t.Fatalf("get BackupJob after reconcile: %v", err)
}
backups := &cnpgtypes.BackupList{}
if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil {
t.Fatalf("list cnpg.io/Backups: %v", err)
}
if len(backups.Items) != 0 {
t.Fatalf("a cnpg.io/Backup was created although the ObjectStore could not be read: %+v", backups.Items)
}
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination = %q, written without knowing whether it changed", got)
}
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("the job failed on a transient read: %s", j.Status.Message)
}
if reconcileErr != nil {
return
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "ClusterPluginApplyRetrying" {
t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying", ready)
}
if res.RequeueAfter == 0 {
t.Errorf("a retried read is not requeued: %+v", res)
}
}
// A change record that cannot be read, or that lies further ahead than
// controller clocks drift, says nothing about when the cache expires, so the
// write replaces it with its own time and the job waits once. It claims the
// destination for that wait, or another job would write its own meanwhile.
func TestReconcileCNPG_UnusableDestinationRecordIsReplaced(t *testing.T) {
for name, value := range map[string]string{
"does not parse": "yesterday",
"far ahead": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
} {
t.Run(name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/")
f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded)
store := f.liveObjectStore()
store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"}
if err := f.c.Update(context.Background(), store); err != nil {
t.Fatalf("seed claim: %v", err)
}
f.setDestinationStamp(value)
j, backups := f.pass()
if len(backups) != 0 {
t.Fatalf("a cnpg.io/Backup was created with an unusable change record: %+v", backups)
}
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("the job failed: %s", j.Status.Message)
}
stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation])
if err != nil || time.Since(stamp) > time.Minute || time.Until(stamp) > time.Minute {
t.Fatalf("%s = %v (%v), want it replaced with the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err)
}
if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" {
t.Errorf("%s = %q, want bj, which waits on the replaced record", cnpgDestinationWrittenByAnnotation, got)
}
})
}
}
// A later write to a Cluster that already carries the plugin re-sends the
// attach record as it is: moving it would make the instances that rolled
// with the plugin look older than the attach.
@@ -5913,9 +6482,11 @@ func TestReconcileCNPG_ArtifactNamesTheDestinationItsBackupWasCreatedWith(t *tes
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.setStrategyDestination("s3://bucket/", "https://s3.one", "creds-one")
f.pass()
f.settleDestination()
j, backups := f.pass()
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
t.Fatalf("got %d cnpg.io/Backups once the destination settled, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
}
if got := f.liveDestination(); got != "s3://bucket/" {
t.Fatalf("destination = %q, want s3://bucket/", got)
@@ -6021,7 +6592,7 @@ func TestReconcileCNPG_ArtifactRefusedForAnUnusableDestinationRecord(t *testing.
"mistyped endpoint": `{"destinationPath":"s3://bucket/","endpointURL":7}`,
} {
t.Run(name, func(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/")
if _, backups := f.pass(); len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups))
}
@@ -6175,7 +6746,7 @@ func TestReconcileCNPG_BackupListErrorIsReturned(t *testing.T) {
// start: a second lookup before the create could only fail, after the write,
// and fail the job with it.
func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/")
f.failingBackupLists(func(own bool, n int) bool { return own && n > 1 })
j, backups := f.pass()