// SPDX-License-Identifier: Apache-2.0 package backupcontroller import ( "context" "errors" "fmt" "slices" "strings" "testing" "time" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" corev1 "k8s.io/api/core/v1" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes" "github.com/cozystack/cozystack/internal/backupcontroller/postgresapp" ) func TestCNPGClusterNameForApp(t *testing.T) { got := cnpgClusterNameForApp("foo") if got != "postgres-foo" { t.Fatalf("cnpgClusterNameForApp(%q) = %q, want postgres-foo", "foo", got) } } func TestBuildBarmanObjectStore_AllFields(t *testing.T) { jobs := int32(4) tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://bucket/path/", EndpointURL: "http://s3:9000", RetentionPolicy: "30d", S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ SecretRef: corev1.LocalObjectReference{Name: "creds"}, }, Wal: &strategyv1alpha1.BarmanWalTemplate{Compression: "gzip"}, Data: &strategyv1alpha1.BarmanDataTemplate{Compression: "gzip", Jobs: &jobs}, } got := buildBarmanObjectStore(tmpl, "my-server") // retentionPolicy lives at spec.backup.retentionPolicy in CNPG; it is // emitted by applyClusterBarmanObjectStore and intentionally absent here. want := &cnpgtypes.BarmanObjectStoreConfiguration{ DestinationPath: "s3://bucket/path/", EndpointURL: "http://s3:9000", ServerName: "my-server", S3Credentials: &cnpgtypes.S3Credentials{ AccessKeyID: &cnpgtypes.SecretKeySelector{Name: "creds", Key: defaultS3AccessKeyIDKey}, SecretAccessKey: &cnpgtypes.SecretKeySelector{Name: "creds", Key: defaultS3SecretAccessKeyKey}, }, Wal: &cnpgtypes.WalBackupConfiguration{Compression: "gzip"}, Data: &cnpgtypes.DataBackupConfiguration{Compression: "gzip", Jobs: new(int32(4))}, } if !apiequality.Semantic.DeepEqual(got, want) { t.Fatalf("buildBarmanObjectStore mismatch\n got: %#v\nwant: %#v", got, want) } } func TestBuildBarmanObjectStore_Minimal(t *testing.T) { tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://only/", } got := buildBarmanObjectStore(tmpl, "fallback") want := &cnpgtypes.BarmanObjectStoreConfiguration{ DestinationPath: "s3://only/", ServerName: "fallback", } if !apiequality.Semantic.DeepEqual(got, want) { t.Fatalf("minimal buildBarmanObjectStore mismatch\n got: %#v\nwant: %#v", got, want) } } func TestBuildBarmanObjectStore_CustomCredentialKeys(t *testing.T) { tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://x/", S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ SecretRef: corev1.LocalObjectReference{Name: "creds"}, AccessKeyIDKey: "AKID", SecretAccessKeyKey: "SKID", }, } got := buildBarmanObjectStore(tmpl, "s") if got.S3Credentials.AccessKeyID.Key != "AKID" { t.Fatalf("expected custom access key id key AKID, got %q", got.S3Credentials.AccessKeyID.Key) } if got.S3Credentials.SecretAccessKey.Key != "SKID" { t.Fatalf("expected custom secret access key key SKID, got %q", got.S3Credentials.SecretAccessKey.Key) } } // TestBuildBarmanObjectStore_EndpointCA covers the TLS-with-self-signed-CA // path: a strategy that names a Secret holding the CA bundle gets translated // into the matching cnpgtypes.SecretKeySelector on the live Cluster, defaulting // the key name to "ca.crt" when not specified. func TestBuildBarmanObjectStore_EndpointCA(t *testing.T) { t.Run("default key", func(t *testing.T) { tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://b/", EndpointCA: &strategyv1alpha1.EndpointCARef{ SecretRef: corev1.LocalObjectReference{Name: "trust-bundle"}, }, } got := buildBarmanObjectStore(tmpl, "s") if got.EndpointCA == nil || got.EndpointCA.Name != "trust-bundle" || got.EndpointCA.Key != "ca.crt" { t.Fatalf("EndpointCA mismatch: %#v", got.EndpointCA) } }) t.Run("custom key", func(t *testing.T) { tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://b/", EndpointCA: &strategyv1alpha1.EndpointCARef{ SecretRef: corev1.LocalObjectReference{Name: "trust-bundle"}, Key: "tls.crt", }, } got := buildBarmanObjectStore(tmpl, "s") if got.EndpointCA == nil || got.EndpointCA.Key != "tls.crt" { t.Fatalf("EndpointCA key not honored: %#v", got.EndpointCA) } }) t.Run("nil block stays nil", func(t *testing.T) { tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"} got := buildBarmanObjectStore(tmpl, "s") if got.EndpointCA != nil { t.Fatalf("EndpointCA must be nil when not configured; got %#v", got.EndpointCA) } }) t.Run("empty SecretRef.Name treated as not configured", func(t *testing.T) { tmpl := strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://b/", EndpointCA: &strategyv1alpha1.EndpointCARef{Key: "ca.crt"}, } got := buildBarmanObjectStore(tmpl, "s") if got.EndpointCA != nil { t.Fatalf("expected nil EndpointCA when SecretRef.Name is empty; got %#v", got.EndpointCA) } }) } func TestRenderCNPGTemplate_TemplatingApplicationName(t *testing.T) { tmpl := strategyv1alpha1.CNPGTemplate{ ServerName: "{{ .Application.metadata.name }}", BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://bucket/{{ .Application.metadata.name }}/", S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ SecretRef: corev1.LocalObjectReference{Name: "{{ .Application.metadata.name }}-creds"}, }, }, } app := newPostgresApp("pg-src", "tenant") got, err := renderCNPGTemplate(tmpl, app, nil) if err != nil { t.Fatalf("renderCNPGTemplate: %v", err) } if got.ServerName != "pg-src" { t.Errorf("ServerName not templated: got %q", got.ServerName) } if got.BarmanObjectStore.DestinationPath != "s3://bucket/pg-src/" { t.Errorf("DestinationPath not templated: got %q", got.BarmanObjectStore.DestinationPath) } if got.BarmanObjectStore.S3Credentials.SecretRef.Name != "pg-src-creds" { t.Errorf("SecretRef.Name not templated: got %q", got.BarmanObjectStore.S3Credentials.SecretRef.Name) } } // TestRenderCNPGTemplate_RoundTripsParametersThroughSnapshot locks in the // fix for review blocker 1 (#4387688473): a strategy template that uses // {{ .Parameters.foo }} for a Secret reference / endpointCA / key name // must yield the same rendered value at restore time as it did at backup // time. Before the fix, the restore reconciler called renderCNPGTemplate // with parameters=nil, so any .Parameters reference rendered "" // and the patched target Postgres app got broken Secret references that // CNPG would later reject. // // Verifies the contract end-to-end: render-backup -> marshal snapshot -> // unmarshal snapshot -> render-restore. Also asserts the buggy path // (parameters=nil at restore) produces a recognisable broken value, so // a regression that drops the persistence side gets caught here too. func TestRenderCNPGTemplate_RoundTripsParametersThroughSnapshot(t *testing.T) { tmpl := strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://bucket/{{ .Application.metadata.name }}/", S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ SecretRef: corev1.LocalObjectReference{Name: "{{ .Parameters.credsSecret }}"}, }, EndpointCA: &strategyv1alpha1.EndpointCARef{ SecretRef: corev1.LocalObjectReference{Name: "{{ .Parameters.caSecret }}"}, Key: "ca.crt", }, }, } app := newPostgresApp("pg-src", "tenant") parameters := map[string]string{ "credsSecret": "tenant-shared-creds", "caSecret": "tenant-shared-ca", } // Backup-time render: real values flow through. rendBackup, err := renderCNPGTemplate(tmpl, app, parameters) if err != nil { t.Fatalf("backup-time render: %v", err) } if rendBackup.BarmanObjectStore.S3Credentials.SecretRef.Name != "tenant-shared-creds" { t.Fatalf("backup-time SecretRef.Name: got %q want %q", rendBackup.BarmanObjectStore.S3Credentials.SecretRef.Name, "tenant-shared-creds") } if rendBackup.BarmanObjectStore.EndpointCA.SecretRef.Name != "tenant-shared-ca" { t.Fatalf("backup-time EndpointCA.SecretRef.Name: got %q want %q", rendBackup.BarmanObjectStore.EndpointCA.SecretRef.Name, "tenant-shared-ca") } // Round-trip parameters through Backup.status.underlyingResources. raw, err := marshalCNPGBackupSnapshot(app, parameters, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: raw}} snap, err := unmarshalCNPGBackupSnapshot(bk) if err != nil { t.Fatalf("unmarshal snapshot: %v", err) } gotParams := snap.Parameters // Restore-time render with retrieved parameters: same values out. rendRestore, err := renderCNPGTemplate(tmpl, app, gotParams) if err != nil { t.Fatalf("restore-time render: %v", err) } if rendRestore.BarmanObjectStore.S3Credentials.SecretRef.Name != "tenant-shared-creds" { t.Errorf("restore-time SecretRef.Name: got %q want %q", rendRestore.BarmanObjectStore.S3Credentials.SecretRef.Name, "tenant-shared-creds") } if rendRestore.BarmanObjectStore.EndpointCA.SecretRef.Name != "tenant-shared-ca" { t.Errorf("restore-time EndpointCA.SecretRef.Name: got %q want %q", rendRestore.BarmanObjectStore.EndpointCA.SecretRef.Name, "tenant-shared-ca") } // Buggy path: rendering at restore with parameters=nil produces // "". Catching this here means a future change that // silently drops parameters off the snapshot fails this test before // it ships a broken patch to a real Postgres app. rendBroken, err := renderCNPGTemplate(tmpl, app, nil) if err != nil { t.Fatalf("nil-params render: %v", err) } if rendBroken.BarmanObjectStore.S3Credentials.SecretRef.Name != "" { t.Errorf("expected broken render for nil params (so the test catches regressions), got %q", rendBroken.BarmanObjectStore.S3Credentials.SecretRef.Name) } } func TestParseCNPGRestoreOptions(t *testing.T) { cases := []struct { name string raw string wantRecoveryTime string wantTimeout int64 wantErr bool }{ {"nil blob", "", "", 0, false}, {"missing fields", `{"foo":"bar"}`, "", 0, false}, {"recovery time only", `{"recoveryTime":"2026-05-01T12:00:00Z"}`, "2026-05-01T12:00:00Z", 0, false}, {"timeout only", `{"restoreTimeoutSeconds":7200}`, "", 7200, false}, {"both", `{"recoveryTime":"2026-05-01T12:00:00Z","restoreTimeoutSeconds":3600}`, "2026-05-01T12:00:00Z", 3600, false}, // A malformed blob must surface a parse error so callers can log it. // Returning silently used to mask "why didn't recoveryTime apply" for // tenants debugging restores; see Blocker 5 in the branch review. {"malformed", `not-json`, "", 0, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var ext *runtime.RawExtension if tc.raw != "" { ext = &runtime.RawExtension{Raw: []byte(tc.raw)} } got, err := parseCNPGRestoreOptions(ext) if tc.wantErr { if err == nil { t.Fatalf("expected parse error, got nil") } return } if err != nil { t.Fatalf("unexpected parse error: %v", err) } if got.RecoveryTime != tc.wantRecoveryTime { t.Errorf("recoveryTime: got %q want %q", got.RecoveryTime, tc.wantRecoveryTime) } if got.RestoreTimeoutSeconds != tc.wantTimeout { t.Errorf("restoreTimeoutSeconds: got %d want %d", got.RestoreTimeoutSeconds, tc.wantTimeout) } }) } } func TestEffectiveRestoreDeadline(t *testing.T) { cases := []struct { name string opts CNPGRestoreOptions want time.Duration }{ {"unset falls back to default", CNPGRestoreOptions{}, cnpgDefaultRestoreDeadline}, {"zero falls back to default", CNPGRestoreOptions{RestoreTimeoutSeconds: 0}, cnpgDefaultRestoreDeadline}, {"negative falls back to default", CNPGRestoreOptions{RestoreTimeoutSeconds: -5}, cnpgDefaultRestoreDeadline}, {"positive override", CNPGRestoreOptions{RestoreTimeoutSeconds: 7200}, 2 * time.Hour}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := tc.opts.effectiveRestoreDeadline() if got != tc.want { t.Errorf("got %s want %s", got, tc.want) } }) } } // TestEffectiveBootstrapDisableGrace pins that the post-convergence // bootstrap-disable window is a FIXED cnpgBootstrapDisableGrace, independent of // restoreTimeoutSeconds: that knob bounds the recovery wait, whereas this window // absorbs a control-plane blip whose duration is unrelated to it. A short timeout // must not shrink the window (a blip would then falsely fail a converged restore // and expose the healthy Cluster to a resubmit's purge-guard), and a long timeout // must not inflate it. func TestEffectiveBootstrapDisableGrace(t *testing.T) { // The grace is a FIXED window, independent of restoreTimeoutSeconds: a short, // a long and an unset recovery timeout all yield the same control-plane-blip // window. Re-coupling it to the deadline turns these red. cases := []struct { name string opts CNPGRestoreOptions }{ {"unset", CNPGRestoreOptions{}}, {"short timeout", CNPGRestoreOptions{RestoreTimeoutSeconds: 60}}, {"long timeout", CNPGRestoreOptions{RestoreTimeoutSeconds: 7200}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := tc.opts.effectiveBootstrapDisableGrace() if got != cnpgBootstrapDisableGrace { t.Errorf("grace %s should be the fixed %s regardless of restoreTimeoutSeconds", got, cnpgBootstrapDisableGrace) } }) } } // TestEffectiveWALArchiveDeadline mirrors TestEffectiveRestoreDeadline for // the dedicated WAL-archive gate cap. The two deadlines exist independently // so a big-DB restore can extend RestoreTimeoutSeconds without simultaneously // loosening the gate that catches a stuck archive_command. func TestEffectiveWALArchiveDeadline(t *testing.T) { cases := []struct { name string opts CNPGRestoreOptions want time.Duration }{ {"unset falls back to default", CNPGRestoreOptions{}, cnpgWALArchiveDeadline}, {"zero falls back to default", CNPGRestoreOptions{WALArchiveTimeoutSeconds: 0}, cnpgWALArchiveDeadline}, {"negative falls back to default", CNPGRestoreOptions{WALArchiveTimeoutSeconds: -5}, cnpgWALArchiveDeadline}, {"positive override", CNPGRestoreOptions{WALArchiveTimeoutSeconds: 600}, 10 * time.Minute}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := tc.opts.effectiveWALArchiveDeadline() if got != tc.want { t.Errorf("got %s want %s", got, tc.want) } }) } } // TestValidateCNPGApplicationRef locks in the API-group guard. Without it a // non-Cozystack ref like other.example.com/Postgres slipped past the Kind // check and hit the hard-wired apps.cozystack.io typed client, reconciling // against the wrong CRD entirely. func TestValidateCNPGApplicationRef(t *testing.T) { cozyGroup := postgresapp.GroupName otherGroup := "other.example.com" cases := []struct { name string ref corev1.TypedLocalObjectReference wantErr bool }{ { name: "kind=Postgres, apiGroup=apps.cozystack.io: accepted", ref: corev1.TypedLocalObjectReference{APIGroup: &cozyGroup, Kind: postgresAppKind, Name: "pg"}, }, { name: "kind=Postgres, nil apiGroup: accepted (default applies upstream)", ref: corev1.TypedLocalObjectReference{Kind: postgresAppKind, Name: "pg"}, }, { name: "wrong kind: rejected", ref: corev1.TypedLocalObjectReference{APIGroup: &cozyGroup, Kind: "MySQL", Name: "pg"}, wantErr: true, }, { name: "right kind, wrong apiGroup: rejected", ref: corev1.TypedLocalObjectReference{APIGroup: &otherGroup, Kind: postgresAppKind, Name: "pg"}, wantErr: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { err := validateCNPGApplicationRef(tc.ref) if tc.wantErr && err == nil { t.Fatalf("expected validation error, got nil") } if !tc.wantErr && err != nil { t.Fatalf("unexpected validation error: %v", err) } }) } } func TestResolveCNPGRestoreTarget(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup src := &backupsv1alpha1.Backup{} src.Namespace = "tenant-foo" src.Spec.ApplicationRef.APIGroup = &apiGroup src.Spec.ApplicationRef.Kind = postgresAppKind src.Spec.ApplicationRef.Name = "pg-src" r := &RestoreJobReconciler{} t.Run("source app when targetApplicationRef nil", func(t *testing.T) { rj := &backupsv1alpha1.RestoreJob{} got := r.resolveCNPGRestoreTarget(rj, src) if got.AppName != "pg-src" || got.Namespace != "tenant-foo" || got.Kind != postgresAppKind { t.Fatalf("unexpected target: %+v", got) } }) t.Run("override app name when targetApplicationRef differs", func(t *testing.T) { rj := &backupsv1alpha1.RestoreJob{} rj.Spec.TargetApplicationRef = newTypedRef(apiGroup, postgresAppKind, "pg-target") got := r.resolveCNPGRestoreTarget(rj, src) if got.AppName != "pg-target" { t.Fatalf("unexpected app name: %q", got.AppName) } }) t.Run("keep source app name when targetApplicationRef matches source", func(t *testing.T) { rj := &backupsv1alpha1.RestoreJob{} rj.Spec.TargetApplicationRef = newTypedRef(apiGroup, postgresAppKind, "pg-src") got := r.resolveCNPGRestoreTarget(rj, src) if got.AppName != "pg-src" { t.Fatalf("unexpected app name: %q", got.AppName) } }) } func TestCNPGBackupPhase(t *testing.T) { cases := []struct { name string backup *cnpgtypes.Backup wantPhase string wantMsg string }{ {"nil", nil, "", ""}, {"completed", &cnpgtypes.Backup{Status: cnpgtypes.BackupStatus{Phase: "completed"}}, "completed", ""}, {"failed-with-error", &cnpgtypes.Backup{Status: cnpgtypes.BackupStatus{Phase: "failed", Error: "boom"}}, "failed", "boom"}, {"empty status", &cnpgtypes.Backup{}, "", ""}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { phase, msg := cnpgBackupPhase(tc.backup) if phase != tc.wantPhase || msg != tc.wantMsg { t.Fatalf("got (%q,%q), want (%q,%q)", phase, msg, tc.wantPhase, tc.wantMsg) } }) } } // newTypedRef is a small helper to build a TypedLocalObjectReference without // the test having to bother with apiGroup pointer plumbing. func newTypedRef(apiGroup, kind, name string) *corev1.TypedLocalObjectReference { return &corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: kind, Name: name} } // newPostgresApp builds a typed Postgres CR for tests. func newPostgresApp(name, namespace string) *postgresapp.Postgres { return &postgresapp.Postgres{ TypeMeta: metav1.TypeMeta{ APIVersion: postgresapp.GroupVersion.String(), Kind: postgresapp.Kind, }, ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, } } // TestBuildPostgresAppRestorePatch_ForwardsSecretRef_NotCleartext locks in // the security-critical contract that S3 credentials are forwarded to the // chart as a Secret reference, not pasted into the CR .spec as cleartext. // Regressing this would expose access keys via etcd, audit logs, and // tenant-readable kubectl get -o yaml. func TestBuildPostgresAppRestorePatch_ForwardsSecretRef_NotCleartext(t *testing.T) { app := newPostgresApp("pg-target", "tenant-foo") creds := &strategyv1alpha1.S3CredentialsTemplate{ SecretRef: corev1.LocalObjectReference{Name: "pg-target-cnpg-backup-creds"}, } patched := buildPostgresAppRestorePatch( app, "pg-src", "postgres-pg-target-restore-deadbeef", "s3://bucket/pg-src/", "https://s3.example", "", "", creds, nil, nil, nil, ) if got := patched.Spec.Backup.S3CredentialsSecret.Name; got != "pg-target-cnpg-backup-creds" { t.Errorf("s3CredentialsSecret.name: got %q want %q", got, "pg-target-cnpg-backup-creds") } // Cleartext key fields must NOT be set. Their presence on the CR .spec is // the bug this test guards against. if got := patched.Spec.Backup.S3AccessKey; got != "" { t.Errorf("spec.backup.s3AccessKey must not be set on the patched app; got %q", got) } if got := patched.Spec.Backup.S3SecretKey; got != "" { t.Errorf("spec.backup.s3SecretKey must not be set on the patched app; got %q", got) } } // TestBuildPostgresAppRestorePatch_ForwardsCustomKeyOverrides verifies the // strategy template's optional key-name overrides round-trip into the chart // values so non-default Secret layouts work end-to-end. func TestBuildPostgresAppRestorePatch_ForwardsCustomKeyOverrides(t *testing.T) { app := newPostgresApp("pg", "tenant") creds := &strategyv1alpha1.S3CredentialsTemplate{ SecretRef: corev1.LocalObjectReference{Name: "creds"}, AccessKeyIDKey: "ACCESS_KEY", SecretAccessKeyKey: "SECRET_KEY", } patched := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "", creds, nil, nil, nil) want := postgresapp.S3CredentialsSecret{ Name: "creds", AccessKeyIDKey: "ACCESS_KEY", SecretAccessKeyKey: "SECRET_KEY", } if got := patched.Spec.Backup.S3CredentialsSecret; got != want { t.Fatalf("s3CredentialsSecret mismatch\n got: %#v\nwant: %#v", got, want) } } // TestBuildPostgresAppRestorePatch_NoSecretRefIsSkipped verifies the // patcher does not stamp an empty s3CredentialsSecret block when the strategy // did not declare credentials. Otherwise the chart would render an empty // Secret name and helm install would fail. func TestBuildPostgresAppRestorePatch_NoSecretRefIsSkipped(t *testing.T) { app := newPostgresApp("pg", "tenant") patched := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "", nil, nil, nil, nil) if got := patched.Spec.Backup.S3CredentialsSecret; got != (postgresapp.S3CredentialsSecret{}) { t.Errorf("spec.backup.s3CredentialsSecret must be zero when credsRef is nil; got %#v", got) } } // TestCNPGPurgeNeeded locks in the dual-guard against re-purging the // freshly-recovered Cluster on a status-update failure. The controller used // to rely solely on a Status condition: if the post-purge Status().Update // raced or failed, the next reconcile would re-purge the just-restored // Cluster, destroying recovery progress. Cross-checking that the live // Cluster is a *freshly-recovered* one (bootstrap.recovery + created after // the job started) makes that second purge a no-op - while still purging a // stale recovery Cluster left over from an earlier completed restore, so a // repeat in-place restore is not a silent no-op. func TestCNPGPurgeNeeded(t *testing.T) { cases := []struct { name string purgedCondition bool liveClusterFreshlyRecovered bool want bool }{ { name: "fresh restore, old cluster still in place: purge", purgedCondition: false, liveClusterFreshlyRecovered: false, want: true, }, { name: "purge already recorded: skip", purgedCondition: true, liveClusterFreshlyRecovered: false, want: false, }, { name: "live cluster freshly recovered by this restore (status write raced): skip", purgedCondition: false, liveClusterFreshlyRecovered: true, want: false, }, { name: "stale recovery cluster from a previous completed restore: purge (repeat-restore fix)", purgedCondition: false, liveClusterFreshlyRecovered: false, want: true, }, { name: "both true: skip (post-purge steady state)", purgedCondition: true, liveClusterFreshlyRecovered: true, want: false, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := cnpgPurgeNeeded(tc.purgedCondition, tc.liveClusterFreshlyRecovered) if got != tc.want { t.Errorf("got %v want %v", got, tc.want) } }) } } // TestCNPGClusterFreshlyRecovered locks in the signal that separates a // recovery Cluster THIS restore just produced from one rendered for another // restore. The repeat-in-place-restore bug was a stale recovery Cluster being // mistaken for a freshly-recovered one and skipping the purge; a Cluster // produced by a concurrent restore, or re-rendered from an earlier restore's // values, is created after StartedAt and must still be purged. func TestCNPGClusterFreshlyRecovered(t *testing.T) { started := metav1.NewTime(time.Now()) after := metav1.NewTime(started.Add(time.Minute)) before := metav1.NewTime(started.Add(-time.Hour)) const ours = "postgres-app-restore-ours" const other = "postgres-app-restore-other" cases := []struct { name string state recoveryClusterState appMark string startedAt *metav1.Time want bool }{ { name: "no recovery bootstrap: not fresh", state: recoveryClusterState{createdAt: &after, restoreMark: ours}, startedAt: &started, want: false, }, { name: "marked by this restore: fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &after, restoreMark: ours}, startedAt: &started, want: true, }, { name: "marked by another restore, created after start (concurrent restore or re-render of an earlier restore): not fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &after, restoreMark: other}, startedAt: &started, want: false, }, { name: "marked by another restore, created before start (leftover from previous restore): not fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &before, restoreMark: other}, startedAt: &started, want: false, }, { name: "marked by this restore with no job start timestamp: fresh, the mark does not need one", state: recoveryClusterState{hasRecovery: true, createdAt: &after, restoreMark: ours}, startedAt: nil, want: true, }, { name: "unmarked, created after start (our own re-render by an older chart): fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &after}, appMark: ours, startedAt: &started, want: true, }, { name: "unmarked, created after start, values not patched by this restore (manual recovery re-rendered): not fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &after}, appMark: "", startedAt: &started, want: false, }, { name: "unmarked, created after start, values patched by another restore: not fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &after}, appMark: other, startedAt: &started, want: false, }, { name: "unmarked, created before start (leftover from previous restore): not fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &before}, appMark: ours, startedAt: &started, want: false, }, { name: "unmarked, created exactly at start: not fresh (conservative tie -> purge)", state: recoveryClusterState{hasRecovery: true, createdAt: &started}, appMark: ours, startedAt: &started, want: false, }, { name: "unmarked, missing cluster creation timestamp: not fresh", state: recoveryClusterState{hasRecovery: true}, appMark: ours, startedAt: &started, want: false, }, { name: "unmarked, missing job start timestamp: not fresh", state: recoveryClusterState{hasRecovery: true, createdAt: &after}, appMark: ours, startedAt: nil, want: false, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := cnpgClusterFreshlyRecovered(tc.state, ours, tc.appMark, tc.startedAt) if got != tc.want { t.Errorf("got %v want %v", got, tc.want) } }) } } // TestBuildPostgresAppRestorePatch_ReplacesTargetUsersAndDatabases locks // in the authoritative-from-source semantics: target's stale users and // databases must be wiped and replaced with source's exact map. The // recovered cluster carries source's role catalog and data; if target's // pre-restore drift survives, the chart's init-job either tries to // re-create roles against the wrong data or resurrects a role a previous // tenant configuration had dropped. func TestBuildPostgresAppRestorePatch_ReplacesTargetUsersAndDatabases(t *testing.T) { app := newPostgresApp("pg-target", "tenant") // Target had pre-existing users/databases (e.g. from a previous restore // or operator drift). Replace must wipe them. app.Spec.Users = map[string]postgresapp.User{ "stale-target-user": {Replication: true}, } app.Spec.Databases = map[string]postgresapp.Database{ "stale-target-db": {Extensions: []string{"pgcrypto"}}, } sourceUsers := map[string]postgresapp.User{ "app": {Replication: true}, } sourceDatabases := map[string]postgresapp.Database{ "appdb": {Extensions: []string{"hstore"}}, } patched := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "", nil, nil, sourceDatabases, sourceUsers) if _, ok := patched.Spec.Users["stale-target-user"]; ok { t.Errorf("stale target user survived restore; replace semantics regressed") } if u, ok := patched.Spec.Users["app"]; !ok { t.Errorf("source user was not propagated onto target") } else if !u.Replication { t.Errorf("source user attributes not propagated; got %#v", u) } if _, ok := patched.Spec.Databases["stale-target-db"]; ok { t.Errorf("stale target database survived restore; replace semantics regressed") } if _, ok := patched.Spec.Databases["appdb"]; !ok { t.Errorf("source database was not propagated onto target") } } // TestBuildPostgresAppRestorePatch_ScrubsStaleBackupSettings guards the // stale-state hygiene called out by coderabbitai on // internal/backupcontroller/cnpgstrategy_controller.go:597-604: a re-restore // into the same target must not inherit a previous restore's recoveryTime, // endpointURL, S3 credentials Secret reference, or inline cleartext S3 keys. func TestBuildPostgresAppRestorePatch_ScrubsStaleBackupSettings(t *testing.T) { app := newPostgresApp("pg-target", "tenant") app.Spec.Bootstrap.RecoveryTime = "2025-01-01T00:00:00Z" app.Spec.Backup.UseSystemBucket = true app.Spec.Backup.EndpointURL = "https://stale.example" app.Spec.Backup.S3AccessKey = "stale-ak" app.Spec.Backup.S3SecretKey = "stale-sk" app.Spec.Backup.S3CredentialsSecret = postgresapp.S3CredentialsSecret{ Name: "stale-creds", AccessKeyIDKey: "OLD_ID", SecretAccessKeyKey: "OLD_KEY", } app.Spec.Backup.EndpointCA = postgresapp.EndpointCA{Name: "stale-ca", Key: "stale.crt"} // Restore with no recoveryTime, no endpointURL, no creds, no CA - // everything stale on the target must be wiped. patched := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "", nil, nil, nil, nil) if got := patched.Spec.Bootstrap.RecoveryTime; got != "" { t.Errorf("stale recoveryTime survived; got %q", got) } if got := patched.Spec.Backup.EndpointURL; got != "" { t.Errorf("stale endpointURL survived; got %q", got) } if got := patched.Spec.Backup.S3AccessKey; got != "" { t.Errorf("inline s3AccessKey was not scrubbed; got %q", got) } if got := patched.Spec.Backup.S3SecretKey; got != "" { t.Errorf("inline s3SecretKey was not scrubbed; got %q", got) } if patched.Spec.Backup.S3CredentialsSecret != (postgresapp.S3CredentialsSecret{}) { t.Errorf("stale s3CredentialsSecret survived; got %+v", patched.Spec.Backup.S3CredentialsSecret) } if patched.Spec.Backup.EndpointCA != (postgresapp.EndpointCA{}) { t.Errorf("stale endpointCA survived; got %+v", patched.Spec.Backup.EndpointCA) } // A system-bucket source must be flipped to the explicit-creds path, or the // chart's bootstrap.enabled + useSystemBucket guard wedges the restore. if patched.Spec.Backup.UseSystemBucket { t.Errorf("useSystemBucket must be cleared on restore so the bootstrap guard passes") } } // TestBuildPostgresAppRestorePatch_ForwardsEndpointCA verifies that a // strategy template with an EndpointCA reference flows through to the // target app's spec.backup.endpointCA. Without this the chart's // externalClusters[].barmanObjectStore.endpointCA stays empty and the // recovery init job panics in CNPG's instance-manager when the S3 // endpoint serves a self-signed certificate (cozystack seaweedfs-s3 does). func TestBuildPostgresAppRestorePatch_ForwardsEndpointCA(t *testing.T) { app := newPostgresApp("pg", "tenant") caRef := &strategyv1alpha1.EndpointCARef{ SecretRef: corev1.LocalObjectReference{Name: "pg-cnpg-backup-ca"}, Key: "ca.crt", } patched := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "", nil, caRef, nil, nil) want := postgresapp.EndpointCA{Name: "pg-cnpg-backup-ca", Key: "ca.crt"} if got := patched.Spec.Backup.EndpointCA; got != want { t.Fatalf("endpointCA mismatch\n got: %#v\nwant: %#v", got, want) } } // TestBuildPostgresAppRestorePatch_ArchiveServerNameDiffersFromSource guards the // in-place restore fix: the restored cluster archives under bootstrap.newServerName, // distinct from the recovery source (bootstrap.serverName / oldName). func TestBuildPostgresAppRestorePatch_ArchiveServerNameDiffersFromSource(t *testing.T) { app := newPostgresApp("pg", "tenant") // An in-place restore is the worst case: source serverName equals the // target cluster's own name. const source = "postgres-pg" patched := buildPostgresAppRestorePatch(app, source, "postgres-pg-restore-deadbeef", "s3://b/", "", "", "", nil, nil, nil, nil) if got := patched.Spec.Bootstrap.ServerName; got != source { t.Errorf("recovery serverName must stay the source; got %q want %q", got, source) } if got := patched.Spec.Bootstrap.OldName; got != source { t.Errorf("recovery oldName must stay the source; got %q want %q", got, source) } if got := patched.Spec.Bootstrap.NewServerName; got == "" || got == source { t.Errorf("archive newServerName must be non-empty and differ from the source %q; got %q", source, got) } } // TestRestoredServerName pins the archive-prefix generator: deterministic per // RestoreJob UID, distinct across UIDs, and never equal to the plain cluster // name (which is the source serverName on an in-place restore). func TestRestoredServerName(t *testing.T) { const cluster = "postgres-pg" a1 := restoredServerName(cluster, types.UID("3c2ee41b-346f-4ae2-98d2-5d629c96f744")) a2 := restoredServerName(cluster, types.UID("3c2ee41b-346f-4ae2-98d2-5d629c96f744")) b := restoredServerName(cluster, types.UID("9eb8feed-7d49-467e-b719-d1896c0602d5")) if a1 != a2 { t.Errorf("not deterministic for a fixed UID: %q vs %q", a1, a2) } if a1 == b { t.Errorf("distinct UIDs must yield distinct serverNames; both %q", a1) } if a1 == cluster { t.Errorf("restored serverName must never equal the plain cluster name %q", cluster) } if !strings.HasPrefix(a1, cluster+"-restore-") { t.Errorf("restored serverName %q must carry the %q-restore- prefix", a1, cluster) } } // TestMarshalUnmarshalCNPGBackupSnapshot round-trips the source-spec // snapshot persisted in Backup.status.underlyingResources, including the // BackupClassStrategy parameters that were in effect at backup time - // without persisting those a strategy that uses {{ .Parameters.foo }} for // a Secret reference / endpointCA / key name would re-render at restore // time with parameters=nil and produce a broken patch. func TestMarshalUnmarshalCNPGBackupSnapshot(t *testing.T) { src := newPostgresApp("pg-src", "tenant") src.Spec.Databases = map[string]postgresapp.Database{ "app": {Extensions: []string{"pgcrypto"}}, } src.Spec.Users = map[string]postgresapp.User{ "app": {Replication: true}, } parameters := map[string]string{ "credsSecret": "tenant-shared-creds", "region": "eu-west-1", } raw, err := marshalCNPGBackupSnapshot(src, parameters, "") if err != nil { t.Fatalf("marshal: %v", err) } if raw == nil { t.Fatalf("expected non-nil RawExtension") } bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: raw}} snap, err := unmarshalCNPGBackupSnapshot(bk) if err != nil { t.Fatalf("unmarshal: %v", err) } dbs, users, params := snap.Databases, snap.Users, snap.Parameters if dbs["app"].Extensions[0] != "pgcrypto" { t.Errorf("databases round-trip mismatch: %#v", dbs) } if !users["app"].Replication { t.Errorf("users round-trip mismatch: %#v", users) } if params["credsSecret"] != "tenant-shared-creds" { t.Errorf("parameters round-trip mismatch: got %#v", params) } if params["region"] != "eu-west-1" { t.Errorf("parameters round-trip mismatch: got %#v", params) } } // TestMarshalUnmarshalCNPGBackupSnapshot_BackwardCompat covers a Backup // taken by an older controller version (no parameters block in the // snapshot JSON). The unmarshal path must treat that as "no parameter // overrides" rather than failing - the strategy template might have used // .Parameters at the time, and we cannot reconstruct values we never // persisted, so the caller proceeds with parameters=nil and lets the // rendered output fall back to whatever the template defaults emit. func TestMarshalUnmarshalCNPGBackupSnapshot_BackwardCompat(t *testing.T) { legacyRaw := []byte(`{"kind":"CNPGBackupSnapshot","apiVersion":"backups.cozystack.io/v1alpha1","databases":{"app":{}},"users":{"app":{}}}`) bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: &runtime.RawExtension{Raw: legacyRaw}}} snap, err := unmarshalCNPGBackupSnapshot(bk) if err != nil { t.Fatalf("unmarshal legacy snapshot: %v", err) } if len(snap.Parameters) != 0 { t.Errorf("legacy snapshot must yield empty parameters, got %#v", snap.Parameters) } // Flavors did not exist when this snapshot was written: every cluster // ran postgresql, and the restore flavor check must read it that way. if got := cnpgFlavor(snap.Flavor); got != "postgresql" { t.Errorf("legacy snapshot must resolve to the postgresql flavor, got %q", got) } } // TestUnmarshalCNPGBackupSnapshot_RejectsMissingOrWrongKind locks in the // fail-loud contract: a Backup without a snapshot must NOT silently // proceed - it would let the chart's init-job DROP recovered roles. Same // for a Backup whose snapshot carries the wrong Kind (foreign payload). func TestUnmarshalCNPGBackupSnapshot_RejectsMissingOrWrongKind(t *testing.T) { cases := []struct { name string ur *runtime.RawExtension }{ {"nil underlyingResources", nil}, {"empty raw", &runtime.RawExtension{Raw: []byte{}}}, {"wrong kind", &runtime.RawExtension{Raw: []byte(`{"kind":"VMInstanceResources","apiVersion":"backups.cozystack.io/v1alpha1"}`)}}, {"malformed json", &runtime.RawExtension{Raw: []byte(`not-json`)}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: tc.ur}} _, err := unmarshalCNPGBackupSnapshot(bk) if err == nil { t.Fatalf("expected error, got nil") } }) } } // TestCNPGBackupDeadlineExceeded locks in the wall-clock cap on waiting for // the barman-cloud plugin rollout, so a rollout that never finishes fails the // BackupJob instead of pinning it in Running. func TestCNPGBackupDeadlineExceeded(t *testing.T) { cases := []struct { name string startedAt *metav1.Time want bool }{ { name: "nil startedAt does not trip the gate", startedAt: nil, want: false, }, { name: "fresh start is well under the deadline", startedAt: &metav1.Time{Time: time.Now()}, want: false, }, { name: "just under deadline does not trip", startedAt: &metav1.Time{Time: time.Now().Add(-(cnpgDefaultBackupDeadline - time.Minute))}, want: false, }, { name: "past deadline trips the gate", startedAt: &metav1.Time{Time: time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))}, want: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := cnpgBackupDeadlineExceeded(tc.startedAt) if got != tc.want { t.Errorf("got %v want %v", got, tc.want) } }) } } // TestCreateCNPGBackupArtifact_AlreadyExistsReturnsExisting locks in the // idempotency guard for createCNPGBackupArtifact: an AlreadyExists collision // (typical when the previous reconcile created the artifact and then raced // on the BackupJob status update) must return the existing artifact rather // than propagate the error and flip a completed run to Failed. func TestCreateCNPGBackupArtifact_AlreadyExistsReturnsExisting(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group preExisting := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"}, DriverMetadata: map[string]string{"marker": "first-reconcile"}, }, } c := newCNPGStrategyTestClient(t, preExisting) r := &BackupJobReconciler{Client: c} j := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, }, } resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"}, } cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}} rendered := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}, } sourceApp := newPostgresApp("pg", "tenant") got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", cnpgDestinationOf(rendered), rendered, sourceApp) if err != nil { t.Fatalf("expected AlreadyExists to be swallowed, got error %v", err) } if got == nil { t.Fatalf("expected existing Backup returned, got nil") } if got.Spec.DriverMetadata["marker"] != "first-reconcile" { t.Errorf("expected the existing object back, got %#v", got.Spec.DriverMetadata) } } // TestApplyClusterPluginBackup_NotFoundOnMissingCluster locks in the // precondition on applyClusterPluginBackup: when the live cnpg.io Cluster has // not yet been rendered by the chart, the driver must surface NotFound (so the // caller can retry-with-backoff) instead of issuing a doomed SSA Apply that // would be rejected by the API server for missing required Cluster fields. func TestApplyClusterPluginBackup_NotFoundOnMissingCluster(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t)} tmpl := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://bucket/x/", }, } _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true, true, "bj") if err == nil { t.Fatalf("expected NotFound error, got nil") } if !apierrors.IsNotFound(err) { t.Fatalf("expected IsNotFound error, got %v", err) } } // TestRecoveryBootstrapClusterState_TerminatingCluster locks in the // DeletionTimestamp guard. Without it, a Cluster CR mid-deletion (after // purgeExistingCluster fired r.Delete but cnpg.io's finalizers haven't // drained yet) would get treated as "still has bootstrap" by the // reconciler. The caller would then either skip the next purge or wait // for healthy on a CR that's about to disappear; in either case Helm // might SSA-merge the chart's bootstrap.recovery onto the terminating // CR and cnpg-operator's bootstrap-immutability check would drop the // change, leaving the cluster on the original initdb spec. func TestRecoveryBootstrapClusterState_TerminatingCluster(t *testing.T) { now := metav1.Now() t.Run("terminating cluster reports not-yet-recovered", func(t *testing.T) { cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "postgres-app", DeletionTimestamp: &now, Finalizers: []string{"cnpg.io/cluster"}, // satisfies finalizer requirement for DeletionTimestamp }, Spec: cnpgtypes.ClusterSpec{ Bootstrap: &cnpgtypes.BootstrapConfiguration{ Recovery: &cnpgtypes.RecoverySource{Source: "pg-src"}, }, }, } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} got, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if got != (recoveryClusterState{}) { t.Fatalf("expected the zero state for a terminating cluster, got %+v", got) } }) t.Run("live recovery cluster reports recovered with creation timestamp", func(t *testing.T) { created := metav1.NewTime(time.Now().Add(-time.Hour)) cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "postgres-app", CreationTimestamp: created, Annotations: map[string]string{cnpgRestoredServerNameAnnotation: "postgres-app-restore-0123456789abcdef"}, }, Spec: cnpgtypes.ClusterSpec{ Bootstrap: &cnpgtypes.BootstrapConfiguration{ Recovery: &cnpgtypes.RecoverySource{Source: "pg-src"}, }, }, } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} got, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if !got.hasRecovery { t.Fatalf("expected hasRecovery for live recovery cluster, got false") } if got.createdAt == nil { t.Fatalf("expected non-nil createdAt for live recovery cluster") } if got.restoreMark != "postgres-app-restore-0123456789abcdef" { t.Fatalf("restoreMark = %q, want the %s annotation", got.restoreMark, cnpgRestoredServerNameAnnotation) } }) t.Run("missing cluster reports not-yet", func(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &RestoreJobReconciler{Client: c} got, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if got != (recoveryClusterState{}) { t.Fatalf("expected the zero state for a missing cluster, got %+v", got) } }) } // TestCNPGBackupWALArchived locks in the gate that defers the destructive // in-place purge until the underlying cnpg.io/Backup is in a state that // guarantees its closing WAL is in object storage. CNPG only flips // Backup.status.phase=completed (and writes endWal) after barman-cloud- // backup has confirmed the upload, so checking the two together is what // the gate uses; an earlier version of this gate compared // Cluster.status.lastArchivedWAL to Backup.status.endWal but // lastArchivedWAL is not present on CNPG 1.27.x cluster status, and the // gate silently treated the missing field as the empty string and never // cleared. func TestCNPGBackupWALArchived(t *testing.T) { const ( ns = "tenant" cluster = "postgres-pg-src" backupID = "pg-src-adhoc" endWal = "000000010000000000000004" ) mkPlatformBackup := func() *backupsv1alpha1.Backup { return &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: backupID}, Spec: backupsv1alpha1.BackupSpec{ DriverMetadata: map[string]string{ cnpgBackupNameKey: backupID, cnpgClusterNameKey: cluster, }, }, } } mkCNPGBackup := func(phase, end string) *cnpgtypes.Backup { return &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: backupID}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: cluster}}, Status: cnpgtypes.BackupStatus{Phase: phase, EndWal: end}, } } t.Run("blocks while underlying cnpg.io/Backup is still running", func(t *testing.T) { c := newCNPGStrategyTestClient(t, mkCNPGBackup("running", "")) r := &RestoreJobReconciler{Client: c} ready, msg, err := r.cnpgBackupWALArchived(context.Background(), mkPlatformBackup()) if err != nil { t.Fatalf("unexpected error: %v", err) } if ready { t.Fatalf("expected not-ready, got ready (%s)", msg) } if msg == "" { t.Errorf("expected human-readable message, got empty") } }) t.Run("blocks when underlying cnpg.io/Backup is in a failure state", func(t *testing.T) { // Failure state must NOT clear the gate even when endWal somehow has // a value - the failure could mean barman is mid-retry and the // closing WAL upload never landed. c := newCNPGStrategyTestClient(t, mkCNPGBackup("failed", endWal)) r := &RestoreJobReconciler{Client: c} ready, _, err := r.cnpgBackupWALArchived(context.Background(), mkPlatformBackup()) if err != nil { t.Fatalf("unexpected error: %v", err) } if ready { t.Fatalf("expected not-ready for phase=failed") } }) t.Run("clears when phase=completed and endWal is set", func(t *testing.T) { c := newCNPGStrategyTestClient(t, mkCNPGBackup(cnpgBackupPhaseComplete, endWal)) r := &RestoreJobReconciler{Client: c} ready, _, err := r.cnpgBackupWALArchived(context.Background(), mkPlatformBackup()) if err != nil { t.Fatalf("unexpected error: %v", err) } if !ready { t.Fatalf("expected ready when phase=completed and endWal is set") } }) t.Run("blocks when phase=completed but endWal is still empty", func(t *testing.T) { // Defensive: if a future CNPG version flips phase to completed // before persisting endWal (or a buggy build does), we want the // gate to stay closed rather than proceeding to a destructive // purge against a backup whose closing WAL we cannot prove is // in object storage. c := newCNPGStrategyTestClient(t, mkCNPGBackup(cnpgBackupPhaseComplete, "")) r := &RestoreJobReconciler{Client: c} ready, _, err := r.cnpgBackupWALArchived(context.Background(), mkPlatformBackup()) if err != nil { t.Fatalf("unexpected error: %v", err) } if ready { t.Fatalf("expected not-ready when endWal is empty") } }) t.Run("conservative on missing cnpg.io/Backup (treat as ready, retention case)", func(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &RestoreJobReconciler{Client: c} ready, _, err := r.cnpgBackupWALArchived(context.Background(), mkPlatformBackup()) if err != nil { t.Fatalf("unexpected error: %v", err) } if !ready { t.Fatalf("expected ready when underlying cnpg.io/Backup is gone (retention)") } }) t.Run("blocks when driverMetadata is missing the backup name", func(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &RestoreJobReconciler{Client: c} platform := mkPlatformBackup() delete(platform.Spec.DriverMetadata, cnpgBackupNameKey) ready, _, err := r.cnpgBackupWALArchived(context.Background(), platform) if err != nil { t.Fatalf("unexpected error: %v", err) } if ready { t.Fatalf("expected not-ready when driverMetadata is missing the backup name") } }) } // TestSetCNPGRestoreHRSuspended locks in the toggle helper used by the // destructive purge to suspend helm-controller before mutating the // Postgres app spec, and to resume it once the live Cluster + PVCs are // gone. Without this guard helm-controller observes the bootstrap.enabled= // true flip while the live Cluster still has bootstrap.initdb, attempts an // upgrade that CNPG admission rejects, and rolls back into a fresh initdb // cluster that races our purge. func TestSetCNPGRestoreHRSuspended(t *testing.T) { const ( ns = "tenant" name = "postgres-app" ) mkHR := func(suspended *bool) *unstructured.Unstructured { hr := &unstructured.Unstructured{} hr.SetGroupVersionKind(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}) hr.SetNamespace(ns) hr.SetName(name) spec := map[string]any{} if suspended != nil { spec["suspend"] = *suspended } _ = unstructured.SetNestedMap(hr.Object, spec, "spec") return hr } suspendedField := func(t *testing.T, dyn dynamic.Interface) bool { t.Helper() hr, err := dyn.Resource(helmReleaseGVR).Namespace(ns).Get(context.Background(), name, metav1.GetOptions{}) if err != nil { t.Fatalf("get HR: %v", err) } got, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend") return got } t.Run("flips spec.suspend false -> true", func(t *testing.T) { falsePtr := false dyn := dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t), mkHR(&falsePtr)) r := &RestoreJobReconciler{Interface: dyn} if err := r.setCNPGRestoreHRSuspended(context.Background(), ns, name, true); err != nil { t.Fatalf("suspend: %v", err) } if !suspendedField(t, dyn) { t.Fatalf("expected spec.suspend=true after suspend call") } }) t.Run("flips spec.suspend true -> false", func(t *testing.T) { truePtr := true dyn := dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t), mkHR(&truePtr)) r := &RestoreJobReconciler{Interface: dyn} if err := r.setCNPGRestoreHRSuspended(context.Background(), ns, name, false); err != nil { t.Fatalf("resume: %v", err) } if suspendedField(t, dyn) { t.Fatalf("expected spec.suspend=false after resume call") } }) t.Run("idempotent when already at desired state", func(t *testing.T) { // Calling suspend on an already-suspended HR (or resume on an // already-resumed one) must be a no-op so retries from a // flapping reconcile loop don't churn HR resourceVersions. truePtr := true dyn := dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t), mkHR(&truePtr)) r := &RestoreJobReconciler{Interface: dyn} if err := r.setCNPGRestoreHRSuspended(context.Background(), ns, name, true); err != nil { t.Fatalf("idempotent suspend: %v", err) } if !suspendedField(t, dyn) { t.Fatalf("expected spec.suspend to stay true") } }) t.Run("missing HR is a no-op", func(t *testing.T) { // A target Postgres app whose chart-rendered HR has not landed // yet (or has been deleted) must not fail the destructive flow: // the controller proceeds with patch+purge and the chart will // later create a fresh HR from the patched values. dyn := dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)) r := &RestoreJobReconciler{Interface: dyn} if err := r.setCNPGRestoreHRSuspended(context.Background(), ns, name, true); err != nil { t.Fatalf("missing HR: %v", err) } }) } // TestCNPGClusterFullyGone locks in the wait-for-purge helper that gates // HR resume until the live Cluster + its PVCs have actually disappeared. // Returning early on a cluster that is mid-deletion (DeletionTimestamp // set, finalizers still draining) lets helm race the still-terminating CR // and we end up back in the "Only one bootstrap" admission-rejection loop. func TestCNPGClusterFullyGone(t *testing.T) { const ( ns = "tenant" name = "postgres-app" ) now := metav1.Now() t.Run("cluster + PVCs absent -> gone=true", func(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &RestoreJobReconciler{Client: c} gone, err := r.cnpgClusterFullyGone(context.Background(), ns, name) if err != nil { t.Fatalf("unexpected error: %v", err) } if !gone { t.Fatalf("expected gone=true with no Cluster and no PVCs") } }) t.Run("live cluster -> gone=false", func(t *testing.T) { cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}, } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} gone, err := r.cnpgClusterFullyGone(context.Background(), ns, name) if err != nil { t.Fatalf("unexpected error: %v", err) } if gone { t.Fatalf("expected gone=false when cluster still exists") } }) t.Run("terminating cluster (DeletionTimestamp) -> gone=false", func(t *testing.T) { cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, DeletionTimestamp: &now, Finalizers: []string{"cnpg.io/cluster"}, }, } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} gone, err := r.cnpgClusterFullyGone(context.Background(), ns, name) if err != nil { t.Fatalf("unexpected error: %v", err) } if gone { t.Fatalf("expected gone=false for terminating cluster") } }) t.Run("cluster gone but PVC remains -> gone=false", func(t *testing.T) { // CNPG's PVCs are deleted alongside the Cluster but storage // drivers (linstor, ceph) can hold them in a Terminating state // for tens of seconds. Resuming the HR while a PVC with the // cluster label still exists lets helm re-render and immediately // re-attach to a leftover volume. pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name + "-1", Labels: map[string]string{cnpgClusterLabel: name}, }, } c := newCNPGStrategyTestClient(t, pvc) r := &RestoreJobReconciler{Client: c} gone, err := r.cnpgClusterFullyGone(context.Background(), ns, name) if err != nil { t.Fatalf("unexpected error: %v", err) } if gone { t.Fatalf("expected gone=false when a labelled PVC is still around") } }) } func TestLogIndicatesRecoveryTargetUnreachable(t *testing.T) { cases := []struct { name string log string want bool }{ { name: "empty log", log: "", want: false, }, { name: "healthy replay progress", log: "LOG: restored log file \"000000010000000000000005\" from archive\nLOG: consistent recovery state reached", want: false, }, { name: "transient recovery-pod crash (API unreachable) does not match", log: `{"level":"error","msg":"while building the manager","error":"failed to get server groups: Get \"https://10.96.0.1:443/api\": dial tcp 10.96.0.1:443: i/o timeout"}`, want: false, }, { name: "target unreachable FATAL matches", log: "LOG: redo done at 0/50000A8\nFATAL: recovery ended before configured recovery target was reached\nLOG: startup process exited with exit code 1", want: true, }, { name: "target unreachable FATAL embedded in JSON log line matches", log: `{"level":"info","record":{"error_severity":"FATAL","message":"recovery ended before configured recovery target was reached"}}`, want: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := logIndicatesRecoveryTargetUnreachable(tc.log); got != tc.want { t.Fatalf("logIndicatesRecoveryTargetUnreachable(%q) = %v, want %v", tc.log, got, tc.want) } }) } } // TestRecoveryUnreachableFromLogs: any recovery log carrying the unreachable- // target FATAL classifies the (already deadline-expired) restore as // target-unreachable; logs without it do not. func TestRecoveryUnreachableFromLogs(t *testing.T) { fatal := "LOG: redo done\nFATAL: " + cnpgRecoveryTargetUnreachableLog healthy := "LOG: restored log file from archive\nLOG: consistent recovery state reached" cases := []struct { name string logs []string want bool }{ {"no logs", nil, false}, {"only healthy/progress logs", []string{healthy, healthy}, false}, {"a FATAL among others", []string{healthy, fatal}, true}, {"all FATAL", []string{fatal, fatal}, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := recoveryUnreachableFromLogs(tc.logs); got != tc.want { t.Fatalf("recoveryUnreachableFromLogs = %v, want %v", got, tc.want) } }) } } // TestRecoveryPodsToInspect pins the filter/order/cap contract of the pod // selection the fail-fast guard relies on: only full-recovery pods, newest // first, capped at cnpgRecoveryMaxInspectPods. A regression here (wrong sort // direction, off-by-one cap, broken container filter) would silently degrade // the fail-fast back into a 30-minute deadline hang. func TestRecoveryPodsToInspect(t *testing.T) { recoveryPod := func(name string, ageSeconds int64) corev1.Pod { return corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, CreationTimestamp: metav1.NewTime(time.Unix(ageSeconds, 0)), }, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: cnpgRecoveryContainerName}}}, } } t.Run("no pods -> empty", func(t *testing.T) { if got := recoveryPodsToInspect(nil); len(got) != 0 { t.Fatalf("expected empty, got %d", len(got)) } }) t.Run("skips non-recovery pods (no full-recovery container)", func(t *testing.T) { pods := []corev1.Pod{ recoveryPod("rec-1", 100), { ObjectMeta: metav1.ObjectMeta{Name: "primary-newest", CreationTimestamp: metav1.NewTime(time.Unix(999, 0))}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "postgres"}}}, }, } got := recoveryPodsToInspect(pods) if len(got) != 1 || got[0].Name != "rec-1" { t.Fatalf("expected only the full-recovery pod, got %v", names(got)) } }) t.Run("newest-first order and cap at cnpgRecoveryMaxInspectPods", func(t *testing.T) { // Seed two more recovery pods than the cap, out of order, plus a // non-recovery pod that is the newest of all (must be excluded). The // result must be exactly the cap-many newest recovery pods, newest-first. n := cnpgRecoveryMaxInspectPods + 2 var pods []corev1.Pod for i := range n { // age i*100 so higher i == newer. pods = append(pods, recoveryPod(fmt.Sprintf("rec-%d", i), int64(i*100))) } pods = append(pods, corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "primary", CreationTimestamp: metav1.NewTime(time.Unix(9_999_999, 0))}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "postgres"}}}, }) got := recoveryPodsToInspect(pods) want := make([]string, 0, cnpgRecoveryMaxInspectPods) for i := range cnpgRecoveryMaxInspectPods { want = append(want, fmt.Sprintf("rec-%d", n-1-i)) // newest-first } if gotNames := names(got); !equalStrings(gotNames, want) { t.Fatalf("expected %v, got %v", want, gotNames) } }) } func names(pods []*corev1.Pod) []string { out := make([]string, len(pods)) for i, p := range pods { out[i] = p.Name } return out } func equalStrings(a, b []string) bool { if len(a) != len(b) { return false } for i := range a { if a[i] != b[i] { return false } } return true } // recoveryTargetUnreachable is nil-safe when no Clientset is wired, so the // deadline stays the backstop instead of the driver panicking. func TestRecoveryTargetUnreachable_NilClientsetReportsFalse(t *testing.T) { r := &RestoreJobReconciler{Client: newCNPGStrategyTestClient(t)} unreachable, pod, forbidden, err := r.recoveryTargetUnreachable(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if unreachable || pod != "" || forbidden { t.Fatalf("expected (false, \"\", false) with no Clientset, got (%v, %q, %v)", unreachable, pod, forbidden) } } // TestRecoveryTargetUnreachable_ClassifiesFromLogs drives the log-based // classification through an injected reader (the fake Clientset's GetLogs // can't return specific content or a Forbidden), covering the three outcomes // the deadline path branches on: the unreachable-target FATAL is present // (-> unreachable, newest pod), it is absent (-> not unreachable), and the log // read is Forbidden (-> forbidden flagged so the caller surfaces the missing // pods/log RBAC instead of failing silently-generic). func TestRecoveryTargetUnreachable_ClassifiesFromLogs(t *testing.T) { const ( ns = "tenant" cluster = "postgres-app" ) recPod := func(name string, ageSeconds int64) client.Object { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, Labels: map[string]string{cnpgClusterLabel: cluster}, CreationTimestamp: metav1.NewTime(time.Unix(ageSeconds, 0)), }, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: cnpgRecoveryContainerName}}}, } } seed := func() []client.Object { return []client.Object{recPod("rec-old", 100), recPod("rec-new", 200)} } fatalLog := "LOG: redo done\nFATAL: " + cnpgRecoveryTargetUnreachableLog t.Run("FATAL present -> unreachable, reports newest pod", func(t *testing.T) { r := &RestoreJobReconciler{ Client: newCNPGStrategyTestClient(t, seed()...), readPodLog: func(context.Context, string, string, string) (string, error) { return fatalLog, nil }, } un, pod, forb, err := r.recoveryTargetUnreachable(context.Background(), ns, cluster) if err != nil { t.Fatalf("unexpected error: %v", err) } if !un || forb || pod != "rec-new" { t.Fatalf("got unreachable=%v pod=%q forbidden=%v, want (true, rec-new, false)", un, pod, forb) } }) t.Run("no FATAL -> not unreachable", func(t *testing.T) { r := &RestoreJobReconciler{ Client: newCNPGStrategyTestClient(t, seed()...), readPodLog: func(context.Context, string, string, string) (string, error) { return "LOG: consistent recovery state reached", nil }, } un, _, forb, err := r.recoveryTargetUnreachable(context.Background(), ns, cluster) if err != nil { t.Fatalf("unexpected error: %v", err) } if un || forb { t.Fatalf("got unreachable=%v forbidden=%v, want (false, false)", un, forb) } }) t.Run("Forbidden log read -> forbidden flagged, not unreachable", func(t *testing.T) { r := &RestoreJobReconciler{ Client: newCNPGStrategyTestClient(t, seed()...), readPodLog: func(context.Context, string, string, string) (string, error) { return "", apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "rec-new", fmt.Errorf("pods/log grant missing")) }, } un, _, forb, err := r.recoveryTargetUnreachable(context.Background(), ns, cluster) if err != nil { t.Fatalf("unexpected error: %v", err) } if un || !forb { t.Fatalf("got unreachable=%v forbidden=%v, want (false, true)", un, forb) } }) } // TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache seeds the cached // and the live client with the Cluster before and after an in-place restore // re-rendered it, and asserts the driver takes both the serverName and the // ObjectStore owner UID from the live one. func TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache(t *testing.T) { archiver := true purged := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", UID: "uid-before-restore"}, } purged.Spec.Plugins = []cnpgtypes.PluginConfiguration{{ Name: cnpgtypes.PluginName, IsWALArchiver: &archiver, Parameters: map[string]string{barmanObjectNameParam: "postgres-app", barmanServerNameParam: "postgres-app"}, }} rebootstrapped := purged.DeepCopy() rebootstrapped.UID = "uid-after-restore" rebootstrapped.Spec.Plugins[0].Parameters[barmanServerNameParam] = "postgres-app-restore-0123456789abcdef" c := newCNPGStrategyTestClient(t, purged) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, rebootstrapped)} tmpl := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"}, } got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj") if err != nil { t.Fatalf("unexpected error: %v", err) } if want := "postgres-app-restore-0123456789abcdef"; got.serverName != want { t.Fatalf("effective serverName = %q, want %q: the restored cluster would archive onto the source's prefix", got.serverName, want) } store := &cnpgtypes.ObjectStore{} if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "postgres-app"}, store); err != nil { t.Fatalf("get ObjectStore after apply: %v", err) } owners := store.GetOwnerReferences() if len(owners) != 1 { t.Fatalf("ObjectStore owner references = %v, want exactly the live Cluster", owners) } if got := string(owners[0].UID); got != "uid-after-restore" { t.Fatalf("ObjectStore owned by UID %q, want the live Cluster's: it would be garbage-collected immediately", got) } } // TestReconcileCNPG_RecordsTheLiveClusterFlavor: the chart refuses a flavor // change on a live Cluster, but the Postgres app keeps the refused value. The // Backup must record the flavor the Cluster actually runs, or a restore would // accept a target of the wrong image family and refuse the right one. func TestReconcileCNPG_RecordsTheLiveClusterFlavor(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group const ns, appName = "tenant", "pg" clusterName := cnpgClusterNameForApp(appName) cases := []struct { name string liveImage string specFlavor string want string }{ {name: "app switched to postgis on a postgresql cluster", liveImage: "ghcr.io/cloudnative-pg/postgresql:17.5", specFlavor: "postgis", want: "postgresql"}, {name: "app switched to postgresql on a postgis cluster", liveImage: "ghcr.io/cloudnative-pg/postgis:17-3.5", specFlavor: "", want: "postgis"}, {name: "cluster on the CNPG default image", liveImage: "", specFlavor: "postgis", want: "postgresql"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { startedAt := metav1.Now() job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, }, Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{ Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }, }, } app := newPostgresApp(appName, ns) app.Spec.Flavor = tc.specFlavor cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, UID: "cluster-uid"}, Spec: cnpgtypes.ClusterSpec{ImageName: tc.liveImage}, } cnpgBackup := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj-cnpg", Labels: map[string]string{ backupsv1alpha1.OwningJobNameLabel: job.Name, backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace, }}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseComplete}, } c := newCNPGStrategyTestClient(t, job, strategy, app, cluster, cnpgBackup) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}, } j := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := r.reconcileCNPG(context.Background(), j, resolved); err != nil { t.Fatalf("reconcileCNPG: %v", err) } if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { t.Fatalf("BackupJob phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) } backup := &backupsv1alpha1.Backup{} if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: job.Name}, backup); err != nil { t.Fatalf("get Backup artifact: %v", err) } snap, err := unmarshalCNPGBackupSnapshot(backup) if err != nil { t.Fatalf("unmarshal snapshot: %v", err) } if got := cnpgFlavor(snap.Flavor); got != tc.want { t.Fatalf("Backup records flavor %q, want the live Cluster's %q", got, tc.want) } }) } } // TestCNPGImageFlavor pins the image-to-flavor mapping to the chart's // postgres.flavorGuard: only the image name counts, never the host, tag or // digest. func TestCNPGImageFlavor(t *testing.T) { cases := map[string]string{ "": "postgresql", "ghcr.io/cloudnative-pg/postgresql:18.1": "postgresql", "ghcr.io/cloudnative-pg/postgis:18-3.6": "postgis", "mirror.local:5000/cloudnative-pg/postgis:18-3.6@sha256:0123": "postgis", "postgis@sha256:0123": "postgis", } for image, want := range cases { if got := cnpgImageFlavor(image); got != want { t.Errorf("cnpgImageFlavor(%q) = %q, want %q", image, got, want) } } } // cnpgDynamicFor builds the dynamic client applyClusterPluginBackup reads // Clusters through, seeded independently of the typed client. func cnpgDynamicFor(t *testing.T, clusters ...*cnpgtypes.Cluster) dynamic.Interface { t.Helper() s := runtime.NewScheme() s.AddKnownTypeWithName(cnpgtypes.GroupVersion.WithKind("Cluster"), &unstructured.Unstructured{}) s.AddKnownTypeWithName(cnpgtypes.GroupVersion.WithKind("ClusterList"), &unstructured.UnstructuredList{}) s.AddKnownTypeWithName(cnpgtypes.BarmanGroupVersion.WithKind("ObjectStore"), &unstructured.Unstructured{}) s.AddKnownTypeWithName(cnpgtypes.BarmanGroupVersion.WithKind("ObjectStoreList"), &unstructured.UnstructuredList{}) objs := make([]runtime.Object, 0, len(clusters)) for _, c := range clusters { raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(c) if err != nil { t.Fatalf("converting Cluster %s: %v", c.Name, err) } u := &unstructured.Unstructured{Object: raw} u.SetAPIVersion(cnpgtypes.GroupVersion.String()) u.SetKind("Cluster") objs = append(objs, u) } return dynamicfake.NewSimpleDynamicClient(s, objs...) } // testCNPGScheme returns a runtime.Scheme that knows the unstructured // HelmRelease GVK used by the dynamic-client tests above. func testCNPGScheme(t *testing.T) *runtime.Scheme { t.Helper() s := runtime.NewScheme() _ = scheme.AddToScheme(s) s.AddKnownTypeWithName(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, &unstructured.Unstructured{}) s.AddKnownTypeWithName(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmReleaseList"}, &unstructured.UnstructuredList{}) return s } // TestApplyClusterPluginBackup_PatchesExistingCluster confirms the happy path: // when the Cluster exists the driver SSA-applies an ObjectStore CR carrying the // The serverName is the S3 WAL-archive path prefix; changing it on a live // Cluster splits the archive across two prefixes and breaks the eventual // restore with "WAL not found". A Cluster that already has the barman-cloud // plugin attached must therefore keep its effective serverName — the explicit // parameter when present, its own name (the plugin's default) when the plugin // is attached without one — and only a Cluster with no plugin at all takes the // strategy template's value. func TestApplyClusterPluginBackup_PreservesLiveServerName(t *testing.T) { explicit := true cases := []struct { name string plugins []cnpgtypes.PluginConfiguration wantServer string }{ { name: "explicit live serverName wins over the strategy's", plugins: []cnpgtypes.PluginConfiguration{{ Name: cnpgtypes.PluginName, IsWALArchiver: &explicit, Parameters: map[string]string{barmanObjectNameParam: "postgres-app", barmanServerNameParam: "postgres-app"}, }}, wantServer: "postgres-app", }, { name: "plugin attached without serverName defaults to the cluster name", plugins: []cnpgtypes.PluginConfiguration{{ Name: cnpgtypes.PluginName, IsWALArchiver: &explicit, Parameters: map[string]string{barmanObjectNameParam: "postgres-app"}, }}, wantServer: "postgres-app", }, { name: "no plugin attached: the strategy's serverName applies", plugins: nil, wantServer: "strategy-name", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", UID: "cluster-uid-123"}, } cluster.Spec.Plugins = tc.plugins c := newCNPGStrategyTestClient(t, cluster) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} tmpl := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"}, } got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj") if err != nil { t.Fatalf("unexpected error: %v", err) } if got.serverName != tc.wantServer { t.Fatalf("effective serverName = %q, want %q", got.serverName, tc.wantServer) } patched := &cnpgtypes.Cluster{} if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "postgres-app"}, patched); err != nil { t.Fatalf("get Cluster after apply: %v", err) } if len(patched.Spec.Plugins) != 1 { t.Fatalf("expected exactly one plugin entry, got %+v", patched.Spec.Plugins) } if s := patched.Spec.Plugins[0].Parameters[barmanServerNameParam]; s != tc.wantServer { t.Fatalf("patched plugin serverName = %q, want %q", s, tc.wantServer) } }) } } // defaultStrategyServerName is the serverName the cozy-default-cnpg Strategy // renders; tests/strategy_cnpg_test.yaml in the backupstrategy-controller chart // pins the chart to this same string. const defaultStrategyServerName = "{{ .Application.metadata.namespace }}-{{ .Application.metadata.name }}{{ with .Application.metadata.uid }}-{{ . }}{{ end }}" func TestRenderCNPGTemplate_DefaultServerName(t *testing.T) { tmpl := strategyv1alpha1.CNPGTemplate{ServerName: defaultStrategyServerName} for _, tc := range []struct { name string uid types.UID want string }{ {name: "uid present", uid: "8f14e45f-ceea-467a-9575-6f0d6e2c1a3b", want: "tenant-pg-8f14e45f-ceea-467a-9575-6f0d6e2c1a3b"}, {name: "uid absent", want: "tenant-pg"}, } { t.Run(tc.name, func(t *testing.T) { app := newPostgresApp("pg", "tenant") app.UID = tc.uid got, err := renderCNPGTemplate(tmpl, app, nil) if err != nil { t.Fatalf("renderCNPGTemplate: %v", err) } if got.ServerName != tc.want { t.Fatalf("serverName = %q, want %q", got.ServerName, tc.want) } }) } } // TestReconcileCNPG_RecreatedAppArchivesUnderAFreshPrefix deletes a Postgres // and recreates it under the same name, then runs a BackupJob against each. // The destinationPath is keyed by name, so only the serverName keeps the new // Cluster from writing its WALs and base backups into the old one's archive; // the empty-archive check that would refuse it is gone by the time the first // BackupJob attaches the plugin. A Cluster that already archives keeps its // serverName, so existing archives stay continuous. func TestReconcileCNPG_RecreatedAppArchivesUnderAFreshPrefix(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group archiver := true backUp := func(t *testing.T, strategyServerName string, appUID types.UID, plugins []cnpgtypes.PluginConfiguration) string { t.Helper() started := metav1.NewTime(time.Now()) job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, }, Status: backupsv1alpha1.BackupJobStatus{StartedAt: &started}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-cnpg"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ ServerName: strategyServerName, BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://bucket/{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}/", }, }}, } app := newPostgresApp("pg", "tenant") app.UID = appUID cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-pg", UID: types.UID("cluster-" + appUID)}, } cluster.Spec.Plugins = plugins c := newCNPGStrategyTestClient(t, job, strategy, app, cluster) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name, }} // A Cluster without the plugin takes two passes: the first only // records that the driver is about to roll it, the second attaches. patched := &cnpgtypes.Cluster{} for range 2 { j := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := r.reconcileCNPG(context.Background(), j, resolved); err != nil { t.Fatalf("reconcileCNPG: %v", err) } if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("BackupJob failed: %+v", j.Status.Conditions) } if err := c.Get(context.Background(), client.ObjectKeyFromObject(cluster), patched); err != nil { t.Fatalf("get Cluster: %v", err) } if got := currentBarmanServerName(patched); got != "" { return got } } t.Fatalf("BackupJob left the barman-cloud plugin unattached: %+v", patched.Spec.Plugins) return "" } for _, tc := range []struct { name string strategyServerName string }{ {name: "default strategy", strategyServerName: defaultStrategyServerName}, {name: "strategy without serverName", strategyServerName: ""}, } { t.Run(tc.name, func(t *testing.T) { first := backUp(t, tc.strategyServerName, "3c2ee41b-346f-4ae2-98d2-5d629c96f744", nil) second := backUp(t, tc.strategyServerName, "9eb8feed-7d49-467e-b719-d1896c0602d5", nil) if first == second { t.Fatalf("both instances of tenant/pg archive under serverName %q: the recreated one writes into its predecessor's archive", first) } for _, name := range []string{first, second} { if strings.Contains(name, "") { t.Fatalf("serverName %q rendered a missing key", name) } } const legacy = "tenant-pg" kept := backUp(t, tc.strategyServerName, "9eb8feed-7d49-467e-b719-d1896c0602d5", []cnpgtypes.PluginConfiguration{{ Name: cnpgtypes.PluginName, IsWALArchiver: &archiver, Parameters: map[string]string{barmanObjectNameParam: "postgres-pg", barmanServerNameParam: legacy}, }}) if kept != legacy { t.Fatalf("a Cluster already archiving under %q moved to %q, splitting its archive", legacy, kept) } }) } } // barman configuration and SSA-patches the Cluster's spec.plugins to reference // it via the barman-cloud plugin (the plugin replaces the deprecated native // spec.backup.barmanObjectStore). serverName must land on the Cluster plugin // parameter, never inside the ObjectStore configuration (the plugin forbids it). func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", UID: "cluster-uid-123"}, } c := newCNPGStrategyTestClient(t, cluster) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} tmpl := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{ DestinationPath: "s3://bucket/x/", RetentionPolicy: "30d", }, } if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true, true, "bj"); err != nil { t.Fatalf("unexpected error: %v", err) } // The ObjectStore CR carries destinationPath + retentionPolicy and no serverName. store := &cnpgtypes.ObjectStore{} if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "postgres-app"}, store); err != nil { t.Fatalf("get ObjectStore after apply: %v", err) } if store.Spec.Configuration.DestinationPath != "s3://bucket/x/" { t.Errorf("ObjectStore destinationPath: got %q", store.Spec.Configuration.DestinationPath) } if store.Spec.RetentionPolicy != "30d" { t.Errorf("ObjectStore retentionPolicy: got %q", store.Spec.RetentionPolicy) } // The ObjectStore must pin the barman-cloud sidecar's S3 request-checksum // policy to when_required, or uploads to non-AWS S3 gateways (Ceph RGW, the // platform's own SeaweedFS system bucket) fail with an x-amz-content-sha256 // InvalidArgument. The chart does not render this ObjectStore in the // platform-managed useSystemBucket=true flow, so this Go path is the only // place that sets it there. sc := store.Spec.InstanceSidecarConfiguration if sc == nil { t.Fatal("ObjectStore has no instanceSidecarConfiguration") } if len(sc.Env) != 1 || sc.Env[0].Name != "AWS_REQUEST_CHECKSUM_CALCULATION" || sc.Env[0].Value != "when_required" { t.Errorf("ObjectStore instanceSidecarConfiguration.env: got %+v, want [AWS_REQUEST_CHECKSUM_CALCULATION=when_required]", sc) } if sc.Resources.Requests.Memory().Value() != 256*1024*1024 || sc.Resources.Limits.Memory().Value() != 1024*1024*1024 || sc.Resources.Requests.Cpu().MilliValue() != 100 { t.Errorf("ObjectStore leaves Barman subject to undersized tenant resource defaults: %+v", sc.Resources) } // Without an explicit CPU limit the tenant LimitRange caps the sidecar at // its 250m default, and every backup and WAL upload compresses on a // quarter of a core. if sc.Resources.Limits.Cpu().MilliValue() != 1000 { t.Errorf("ObjectStore leaves the sidecar CPU limit to the tenant LimitRange default: %+v", sc.Resources.Limits) } // The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC // removes it when the Cluster is deleted (no orphan in the platform flow, // where the chart does not render this ObjectStore). if len(store.OwnerReferences) != 1 { t.Fatalf("expected exactly one ownerReference on the ObjectStore, got %+v", store.OwnerReferences) } own := store.OwnerReferences[0] if own.Kind != "Cluster" || own.Name != "postgres-app" || own.UID != "cluster-uid-123" { t.Errorf("ObjectStore ownerReference: got kind=%q name=%q uid=%q, want Cluster/postgres-app/cluster-uid-123", own.Kind, own.Name, own.UID) } if store.Spec.Configuration.ServerName != "" { t.Errorf("ObjectStore configuration must not carry serverName (plugin forbids it); got %q", store.Spec.Configuration.ServerName) } // The Cluster references the ObjectStore through spec.plugins, with // serverName on the plugin parameter and isWALArchiver=true. got := &cnpgtypes.Cluster{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(cluster), got); err != nil { t.Fatalf("get cluster after patch: %v", err) } if len(got.Spec.Plugins) != 1 { t.Fatalf("expected exactly one plugin entry, got %+v", got.Spec.Plugins) } p := got.Spec.Plugins[0] if p.Name != cnpgtypes.PluginName { t.Errorf("plugin name: got %q, want %q", p.Name, cnpgtypes.PluginName) } if p.IsWALArchiver == nil || !*p.IsWALArchiver { t.Errorf("expected isWALArchiver=true, got %v", p.IsWALArchiver) } if got := p.Parameters[barmanObjectNameParam]; got != "postgres-app" { t.Errorf("plugin barmanObjectName: got %q, want %q", got, "postgres-app") } if got := p.Parameters[barmanServerNameParam]; got != "tenant-app" { t.Errorf("plugin serverName: got %q, want %q", got, "tenant-app") } } // TestEnsureCNPGBackup_UsesPluginMethod locks in that driver-initiated backups // go through the barman-cloud plugin. spec.method / spec.pluginConfiguration is // the single field that routes the run: if it regressed to the default // (barmanObjectStore) method, CNPG would run the legacy method against a Cluster // that no longer has one and every platform backup would fail. func TestEnsureCNPGBackup_UsesPluginMethod(t *testing.T) { j := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, } c := newCNPGStrategyTestClient(t, j) r := &BackupJobReconciler{Client: c} bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", nil, cnpgDestination{DestinationPath: "s3://bucket/"}) if err != nil { t.Fatalf("ensureCNPGBackup: %v", err) } if bk.Spec.Method != cnpgtypes.BackupMethodPlugin { t.Errorf("Backup spec.method: got %q, want %q", bk.Spec.Method, cnpgtypes.BackupMethodPlugin) } if bk.Spec.PluginConfiguration == nil || bk.Spec.PluginConfiguration.Name != cnpgtypes.PluginName { t.Errorf("Backup spec.pluginConfiguration: got %+v, want name=%q", bk.Spec.PluginConfiguration, cnpgtypes.PluginName) } if bk.Spec.Cluster.Name != "postgres-app" { t.Errorf("Backup spec.cluster.name: got %q, want postgres-app", bk.Spec.Cluster.Name) } } // TestReconcileCNPG_StartedAtPreservedAcrossStaleCache locks in the bug fix // for the StartedAt race called out in the branch review: a second reconcile // with a stale local copy (which observes StartedAt==nil) must NOT advance // the persisted timestamp the first reconcile already wrote. Without the // refetch-before-write guard, the wall-clock deadline gate slides forward // on every poll, defeating the purpose of cnpgDefaultBackupDeadline. func TestReconcileCNPG_StartedAtPreservedAcrossStaleCache(t *testing.T) { // metav1.Time loses sub-second precision on JSON round-trip; truncate // upfront so the equality check below isn't fooled by nanosecond drift. original := metav1.NewTime(time.Now().Add(-10 * time.Minute).Truncate(time.Second)) stored := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Status: backupsv1alpha1.BackupJobStatus{ StartedAt: &original, }, } c := newCNPGStrategyTestClient(t, stored) r := &BackupJobReconciler{Client: c} // Stale local copy as the reconcile would have observed it: StartedAt==nil. stale := stored.DeepCopy() stale.Status.StartedAt = nil // Inline the refetch-before-write block from reconcileCNPG. (The full // reconciler path requires more wiring than this lightweight test // builds; the block under test is the bug-fix surface, not the // surrounding decode/template logic.) fresh := &backupsv1alpha1.BackupJob{} if err := r.Get(context.Background(), client.ObjectKeyFromObject(stored), fresh); err != nil { t.Fatalf("get fresh: %v", err) } if fresh.Status.StartedAt != nil { stale.Status.StartedAt = fresh.Status.StartedAt } else { base := fresh.DeepCopy() now := metav1.Now() fresh.Status.StartedAt = &now if err := r.Status().Patch(context.Background(), fresh, client.MergeFrom(base)); err != nil { t.Fatalf("patch: %v", err) } stale.Status.StartedAt = fresh.Status.StartedAt } got := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(stored), got); err != nil { t.Fatalf("get after fix: %v", err) } if got.Status.StartedAt == nil || !got.Status.StartedAt.Equal(&original) { t.Errorf("persisted StartedAt was advanced by stale-cache reconcile; got %v, want %v", got.Status.StartedAt, original) } if stale.Status.StartedAt == nil || !stale.Status.StartedAt.Equal(&original) { t.Errorf("local copy did not pick up persisted StartedAt; got %v, want %v", stale.Status.StartedAt, original) } } // TestReconcileCNPG_StartedAtReturnsEarlyToAvoidStaleResourceVersion locks // in the early-return after the StartedAt patch. Before the fix, reconcileCNPG // patched StartedAt onto a refetched copy and then continued the same call // using the local `j` whose ResourceVersion was stale, so any subsequent // r.Status().Update on `j` (e.g. the markBackupJobFailed path or the // ClusterNotReady condition write) carried a pre-patch RV and produced a // Conflict on the very first poll. The fix returns RequeueAfter immediately // after the StartedAt patch so the next reconcile reads `j` cleanly. func TestReconcileCNPG_StartedAtReturnsEarlyToAvoidStaleResourceVersion(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group stored := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, }, } c := newCNPGStrategyTestClient(t, stored) r := &BackupJobReconciler{Client: c} j := stored.DeepCopy() resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "missing-strategy", }, } result, err := r.reconcileCNPG(context.Background(), j, resolved) if err != nil { t.Fatalf("reconcileCNPG: unexpected error %v", err) } if result.RequeueAfter == 0 { t.Fatalf("expected RequeueAfter > 0 after StartedAt patch, got %+v", result) } persisted := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(stored), persisted); err != nil { t.Fatalf("get persisted: %v", err) } if persisted.Status.StartedAt == nil { t.Errorf("expected StartedAt persisted, got nil") } // Without the early return, reconcileCNPG would have proceeded past the // StartedAt block, found the strategy missing, and called // markBackupJobFailed - persisting Phase=Failed (and, depending on RV // enforcement, returning Conflict). With the fix, Phase stays empty and // the next reconcile picks up cleanly with the post-patch RV. if persisted.Status.Phase != "" { t.Errorf("expected Phase unchanged after StartedAt early return, got %q", persisted.Status.Phase) } } // CloudNativePG never reconciles a Backup out of a terminal phase (the // backup controller returns early on failed and completed), so a failed // Backup fails the BackupJob at once. Any other phase keeps the BackupJob // waiting past the backup deadline: CNPG keeps a healthy Backup pending // while another backup of the Cluster runs or while it restarts one. func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group const ns, appName = "tenant", "pg" clusterName := cnpgClusterNameForApp(appName) expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) cases := []struct { name string phase string errText string startedAt time.Time wantPhase backupsv1alpha1.BackupJobPhase wantMessage []string notInMessage []string }{ {name: "failed fails at once", phase: cnpgBackupPhaseFailed, errText: "rpc error: code = Unknown desc = exit status 1", startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "exit status 1", "plugin-barman-cloud"}}, {name: "failed without an error text fails at once", phase: cnpgBackupPhaseFailed, startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "no error recorded"}, notInMessage: []string{"plugin-barman-cloud"}}, {name: "failed before the plugin ran points nowhere else", phase: cnpgBackupPhaseFailed, errText: "cannot backup a hibernated cluster", startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "hibernated"}, notInMessage: []string{"plugin-barman-cloud"}}, {name: "pending keeps waiting", phase: "pending", startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}}, {name: "pending is not capped by the backup deadline", phase: "pending", startedAt: expired, wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}}, {name: "no phase yet keeps waiting", phase: "", startedAt: expired, wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg"}}, {name: "started keeps running", phase: "started", startedAt: expired, wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "started"}}, {name: "walArchivingFailing keeps waiting", phase: "walArchivingFailing", startedAt: expired, wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "walArchivingFailing"}}, {name: "completed succeeds", phase: cnpgBackupPhaseComplete, startedAt: expired, wantPhase: backupsv1alpha1.BackupJobPhaseSucceeded}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { startedAt := metav1.NewTime(tc.startedAt) job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, }, Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{ Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }, }, } cluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, UID: "cluster-uid"}, } cnpgBackup := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj-cnpg", Labels: map[string]string{ backupsv1alpha1.OwningJobNameLabel: job.Name, backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace, }}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, Status: cnpgtypes.BackupStatus{Phase: tc.phase, Error: tc.errText}, } c := newCNPGStrategyTestClient(t, job, strategy, newPostgresApp(appName, ns), cluster, cnpgBackup) r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}, } j := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil { t.Fatalf("get BackupJob: %v", err) } res, err := r.reconcileCNPG(context.Background(), j, resolved) if err != nil { t.Fatalf("reconcileCNPG: %v", err) } if j.Status.Phase != tc.wantPhase { t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) } stored := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), stored); err != nil { t.Fatalf("get BackupJob after reconcile: %v", err) } if stored.Status.Phase != tc.wantPhase { t.Fatalf("stored BackupJob phase = %q, want %q", stored.Status.Phase, tc.wantPhase) } message := j.Status.Message if tc.wantPhase == backupsv1alpha1.BackupJobPhaseRunning { if res.RequeueAfter == 0 { t.Errorf("expected a requeue while the cnpg.io/Backup is in progress, got %+v", res) } persisted := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), persisted); err != nil { t.Fatalf("get BackupJob after reconcile: %v", err) } ready := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForCNPGBackup" { t.Fatalf("expected Ready=False/WaitingForCNPGBackup while waiting, got %+v", ready) } message = ready.Message // A pass that observes the same phase again must not write the // BackupJob status: the driver polls every few seconds. statusWrites := 0 r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { statusWrites++ return cl.SubResource(sub).Update(ctx, obj, opts...) }, SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { statusWrites++ return cl.SubResource(sub).Patch(ctx, obj, patch, opts...) }, }) if _, err := r.reconcileCNPG(context.Background(), persisted, resolved); err != nil { t.Fatalf("second reconcileCNPG: %v", err) } if statusWrites != 0 { t.Errorf("an unchanged phase wrote the BackupJob status %d time(s)", statusWrites) } } for _, want := range tc.wantMessage { if !strings.Contains(message, want) { t.Errorf("message %q does not mention %q", message, want) } } for _, unwanted := range tc.notInMessage { if strings.Contains(message, unwanted) { t.Errorf("message %q mentions %q", message, unwanted) } } artifacts := &backupsv1alpha1.BackupList{} if err := c.List(context.Background(), artifacts, client.InNamespace(ns)); err != nil { t.Fatalf("list Backups: %v", err) } wantArtifacts := 0 if tc.wantPhase == backupsv1alpha1.BackupJobPhaseSucceeded { wantArtifacts = 1 } if len(artifacts.Items) != wantArtifacts { t.Errorf("got %d Backup artifacts, want %d", len(artifacts.Items), wantArtifacts) } }) } } // newAttachedObjectStoreFixture is a Cluster that already runs the barman-cloud // plugin, whose ObjectStore currently points at liveDestination; the strategy // of the fixture's BackupJob points at s3://bucket/. func newAttachedObjectStoreFixture(t *testing.T, startedAt time.Time, liveDestination string) *cnpgRolloutFixture { t.Helper() f := newCNPGRolloutFixture(t, newRolloutBackupJob(startedAt), newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(time.Now().Add(-time.Hour))) store := &cnpgtypes.ObjectStore{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster}, Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: liveDestination}}, } if err := f.c.Create(context.Background(), store); err != nil { t.Fatalf("seed ObjectStore: %v", err) } return f } func (f *cnpgRolloutFixture) liveDestination() string { f.t.Helper() store := &cnpgtypes.ObjectStore{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil { f.t.Fatalf("get ObjectStore: %v", err) } return store.Spec.Configuration.DestinationPath } // seedCNPGBackup creates a cnpg.io/Backup of cluster owned by the BackupJob // named owner, labelled as ensureCNPGBackup labels it; an empty owner leaves // it unlabelled, as a Backup created by hand. func (f *cnpgRolloutFixture) seedCNPGBackup(name, owner, cluster, phase string) { f.t.Helper() var backupLabels map[string]string if owner != "" { backupLabels = map[string]string{ backupsv1alpha1.OwningJobNameLabel: owner, backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS, cnpgClusterLabel: cluster, } } b := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: name, Labels: backupLabels}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: cluster}}, Status: cnpgtypes.BackupStatus{Phase: phase}, } if err := f.c.Create(context.Background(), b); err != nil { f.t.Fatalf("seed cnpg.io/Backup %s: %v", name, err) } } func (f *cnpgRolloutFixture) setJobCondition(cond metav1.Condition) { f.t.Helper() j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { f.t.Fatalf("get BackupJob: %v", err) } j.Status.Conditions = append(j.Status.Conditions, cond) if err := f.c.Status().Update(context.Background(), j); err != nil { f.t.Fatalf("seed BackupJob condition: %v", err) } } func (f *cnpgRolloutFixture) seedOtherBackupJob(name string, phase backupsv1alpha1.BackupJobPhase) { f.t.Helper() other := newRolloutBackupJob(time.Now().Add(-time.Hour)) other.Name = name other.Status.Phase = phase if err := f.c.Create(context.Background(), other); err != nil { f.t.Fatalf("seed BackupJob %s: %v", name, err) } } // The ObjectStore is the Cluster's WAL archive destination, so a BackupJob // whose cnpg.io/Backup is already running must not keep re-applying it: a // second BackupJob of the same app with another destination would otherwise // flip it back and forth under both backups. func TestReconcileCNPG_RunningBackupDoesNotReapplyObjectStore(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") // Two jobs can still pass the queue together, across a leader handover; // once both have a Backup neither may wait on the other. f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") writes := 0 f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { writes++ return cl.Patch(ctx, obj, patch, opts...) }, }) j, _ := f.pass() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("BackupJob failed: %s", j.Status.Message) } if got := f.liveDestination(); got != "s3://other/" { t.Fatalf("a pass over a running backup re-applied the ObjectStore: destination %q", got) } if writes != 0 { t.Errorf("a pass over a running backup patched %d object(s); the ObjectStore and the Cluster plugin are only read", writes) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { t.Fatalf("Ready = %+v, want the running backup followed rather than queued", ready) } } // The queue reads the apiserver on every pass of every waiting job, so it // asks only for the Backups this controller created for the Cluster rather // than the namespace's whole Backup history. func TestReconcileCNPG_QueueListsOnlyTheClustersOwnedBackups(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") var selectors []labels.Selector f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*cnpgtypes.BackupList); ok { lo := &client.ListOptions{} lo.ApplyOptions(opts) selectors = append(selectors, lo.LabelSelector) } return cl.List(ctx, list, opts...) }, }) f.pass() owned := labels.Set{cnpgClusterLabel: rolloutCluster, backupsv1alpha1.OwningJobNameLabel: "bj-other"} foreignCluster := labels.Set{cnpgClusterLabel: "postgres-other", backupsv1alpha1.OwningJobNameLabel: "bj-other"} unowned := labels.Set{cnpgClusterLabel: rolloutCluster} var queue labels.Selector for _, s := range selectors { if s == nil { s = labels.Everything() } if s.Matches(owned) { queue = s } } if queue == nil { t.Fatalf("no cnpg.io/Backup list would find another job's Backup; selectors = %v", selectors) } if queue.Matches(foreignCluster) || queue.Matches(unowned) { t.Errorf("queue selector %q, want it to match only owned Backups of %s", queue, rolloutCluster) } } func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { cases := []struct { name string otherJob backupsv1alpha1.BackupJobPhase seedJob bool otherBackup string owner string cluster string wantWait bool }{ {name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true}, {name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true}, {name: "a completed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseComplete}, {name: "a failed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseFailed}, {name: "a Backup of a failed job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "pending"}, {name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"}, {name: "a Backup whose job is gone does not", otherBackup: "pending"}, {name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"}, {name: "a Backup created by hand does not", otherBackup: "running", owner: "-"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") if tc.seedJob { f.seedOtherBackupJob("bj-other", tc.otherJob) } owner, cluster := "bj-other", rolloutCluster if tc.owner == "-" { owner = "" } if tc.cluster != "" { cluster = tc.cluster } f.seedCNPGBackup("bj-other-cnpg", owner, cluster, tc.otherBackup) // client-go refuses a Get with an empty name before it reaches // the apiserver; the fake client answers NotFound instead. statusWrites := 0 f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if key.Name == "" { return errors.New("resource name may not be empty") } return cl.Get(ctx, key, obj, opts...) }, SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { statusWrites++ return cl.SubResource(sub).Update(ctx, obj, opts...) }, }) j, backups, res := f.passResult() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("BackupJob failed: %s", j.Status.Message) } ours := 0 for _, b := range backups { if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { ours++ } } ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready") if tc.wantWait { if got := f.liveDestination(); got != "s3://other/" { t.Errorf("the waiting job applied its destination over a running backup: %q", got) } if ours != 0 { t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours) } if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready) } if res.RequeueAfter == 0 { t.Errorf("a queued job is not requeued, and nothing else wakes it when the holder finishes: %+v", res) } statusWrites = 0 if _, _, res = f.passResult(); res.RequeueAfter == 0 { t.Errorf("a queued job whose status did not change is not requeued: %+v", res) } if statusWrites != 0 { t.Errorf("a second pass with the same holder wrote the BackupJob status %d time(s)", statusWrites) } held := &cnpgtypes.Backup{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { t.Fatalf("get holder Backup: %v", err) } held.Status.Phase = cnpgBackupPhaseComplete if err := f.c.Update(context.Background(), held); err != nil { t.Fatalf("complete holder Backup: %v", err) } j, _ = f.pass() if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { t.Errorf("Ready still names the finished holder: %+v", ready) } return } if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, want the job's own s3://bucket/", got) } if ready != nil && ready.Reason == "WaitingForOtherBackup" { t.Errorf("Ready = %+v, the job was held", ready) } f.settleDestination() _, backups = f.pass() ours = 0 for _, b := range backups { if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { ours++ } } if ours != 1 { t.Errorf("got %d cnpg.io/Backups for the job, want 1", ours) } }) } } func TestReconcileCNPG_ObjectStoreApplyErrors(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) webhookDown := apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io": connect: connection refused`)) invalid := apierrors.NewInvalid(schema.GroupKind{Group: cnpgtypes.BarmanGroupVersion.Group, Kind: "ObjectStore"}, rolloutCluster, nil) cases := []struct { name string err error startedAt time.Time // failingSince seeds an earlier failure of the same write. failingSince time.Time wantPhase backupsv1alpha1.BackupJobPhase }{ {name: "a transient error is retried", err: webhookDown, startedAt: time.Now()}, {name: "a transient error failing since past the deadline fails", err: webhookDown, startedAt: expired, failingSince: expired, wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, {name: "a transient error after a long wait is retried", err: webhookDown, startedAt: expired}, {name: "an invalid object fails at once", err: invalid, startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, {name: "a bad request fails at once", err: apierrors.NewBadRequest("malformed"), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, {name: "a forbidden write fails at once", err: apierrors.NewForbidden(schema.GroupResource{Resource: "objectstores"}, rolloutCluster, errors.New("denied")), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, tc.startedAt, "s3://other/") if !tc.failingSince.IsZero() { f.setJobCondition(metav1.Condition{ Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", Message: "earlier failure", LastTransitionTime: metav1.NewTime(tc.failingSince), }) } f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok { return tc.err } return cl.Patch(ctx, obj, patch, opts...) }, }) j, backups, res := f.passResult() if j.Status.Phase != tc.wantPhase { t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) } if len(backups) != 0 { t.Errorf("a cnpg.io/Backup was created although the ObjectStore was not applied: %+v", backups) } if tc.wantPhase == "" { ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready") if ready == nil || ready.Reason != "ClusterPluginApplyRetrying" || !strings.Contains(ready.Message, "connection refused") { t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying carrying the error", ready) } if res.RequeueAfter == 0 { t.Errorf("a retried write is not requeued: %+v", res) } } }) } } // A write that recovers clears the retry state, so a later failure gets a // fresh budget and Ready no longer reports the retry. func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") failing := true f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok && failing { return apierrors.NewInternalError(errors.New("webhook down")) } return cl.Patch(ctx, obj, patch, opts...) }, }) j, _ := f.pass() if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond == nil || cond.Status != metav1.ConditionFalse { t.Fatalf("%s = %+v after a failed write, want False", backupCondDestinationApplied, cond) } failing = false j, _ = f.pass() f.settleDestination() j, backups := f.pass() if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups after the write recovered, want 1", len(backups)) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { t.Errorf("%s = %+v left over after the write recovered", backupCondDestinationApplied, cond) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "ClusterPluginApplyRetrying" { t.Errorf("Ready still reports the retry: %+v", ready) } } // An error reading the holder's BackupJob is not taken as "no holder". func TestReconcileCNPG_HolderLookupErrorIsReturned(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if key.Name == "bj-other" { return apierrors.NewServiceUnavailable("cache not synced") } return cl.Get(ctx, key, obj, opts...) }, }) j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { t.Fatalf("reconcileCNPG swallowed the holder lookup error") } if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination written although the holder could not be read: %q", got) } } // The rollout deadline counts from when the plugin was attached, not from // the job's start, which may lie behind a long wait for another backup. func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired), newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) f.pass() // latch j, _ := f.pass() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("a rollout that just began failed on the job's old start: %s", j.Status.Message) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" { t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond) } attached := &cnpgtypes.Cluster{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { t.Fatalf("get Cluster: %v", err) } f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) rollout.LastTransitionTime = metav1.NewTime(expired) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the rollout: %v", err) } j, _ = f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { t.Fatalf("phase = %q (%s), want Failed on a rollout older than the deadline", j.Status.Phase, j.Status.Message) } } // A rollout dated by the Cluster's attach record runs out like one dated by // the job's own condition: a record newer than the condition but older than // the deadline fails the job too. func TestReconcileCNPG_PluginRolloutDeadlineCountsFromTheAttachRecord(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(expired.Add(-time.Hour))) j, _ := f.pass() // latch latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) if latch == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } latch.LastTransitionTime = metav1.NewTime(expired.Add(-10 * time.Minute)) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the latch: %v", err) } live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: expired.UTC().Format(time.RFC3339)} f.observeCNPG(live) j, backups := f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups) } if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { t.Fatalf("phase = %q (%s), want Failed on a rollout recorded longer ago than the deadline", j.Status.Phase, j.Status.Message) } } // The rollout is measured from the write that attached the plugin, not from // the latch recorded before it: retries of a failing write can lie between // the two, and an instance recreated in that time runs without the plugin. func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) { cases := []struct { name string // podBetween recreates the instance after the latch and before the // attach; otherwise it predates both. podBetween bool }{ {name: "an instance recreated before the attach still has to roll", podBetween: true}, {name: "retries before the attach do not spend the rollout deadline"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) oldPod := newRolloutInstancePod(expired.Add(-time.Hour)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), oldPod) j, _ := f.pass() // latch latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) if latch == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } latch.LastTransitionTime = metav1.NewTime(expired) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the latch: %v", err) } failing := true f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.Cluster); ok && failing { return apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io"`)) } return cl.Patch(ctx, obj, patch, opts...) }, }) if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("a failed attach failed the job: %s", j.Status.Message) } if tc.podBetween { f.replacePod(oldPod, newRolloutInstancePod(expired.Add(time.Minute))) } failing = false attachAt := time.Now().Truncate(time.Second) j, backups := f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created in the pass that attached the plugin: %+v", backups) } rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) } attached := &cnpgtypes.Cluster{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { t.Fatalf("get Cluster: %v", err) } f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) j, backups = f.pass() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("a rollout that began with the attach failed: %s", j.Status.Message) } if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || !strings.Contains(ready.Message, "predates") { t.Errorf("Ready = %+v, want the instance that predates the plugin", ready) } }) } } // The status write that records the attach can be lost after the attach // itself went through, by a conflict, an apiserver that is briefly // unavailable, or a restart, including on the retry of an attach that // failed; an instance recreated between the latch and the attach still has // to roll, and the rollout deadline still counts from the attach. func TestReconcileCNPG_LostAttachRecordStillWaitsForTheRollout(t *testing.T) { jobs := backupsv1alpha1.GroupVersion.WithResource("backupjobs").GroupResource() for _, tc := range []struct { name string failFirstAttach bool statusWriteError error }{ {name: "conflict", statusWriteError: apierrors.NewConflict(jobs, "bj", errors.New("the object has been modified"))}, {name: "retried attach, apiserver unavailable", failFirstAttach: true, statusWriteError: apierrors.NewServiceUnavailable("the server is currently unable to handle the request")}, } { t.Run(tc.name, func(t *testing.T) { latchedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) oldPod := newRolloutInstancePod(latchedAt.Add(-time.Hour)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), oldPod) j, _ := f.pass() // latch latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) if latch == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } latch.LastTransitionTime = metav1.NewTime(latchedAt) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the latch: %v", err) } f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(time.Minute))) base := f.r.Client if tc.failFirstAttach { f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.Cluster); ok { return apierrors.NewServiceUnavailable("the server is currently unable to handle the request") } return cl.Patch(ctx, obj, patch, opts...) }, }) if j, _ = f.pass(); apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) { t.Fatalf("a failed attach reported the rollout done: %+v", j.Status.Conditions) } if applied := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); applied == nil || applied.Reason != "ClusterPluginApplyRetrying" { t.Fatalf("%s = %+v, want the attach retrying", backupCondDestinationApplied, applied) } } f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { return tc.statusWriteError }, }) j = &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { t.Fatalf("the pass that attached the plugin reported no lost status write") } f.r.Client = base attached := &cnpgtypes.Cluster{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { t.Fatalf("get Cluster: %v", err) } if len(attached.Spec.Plugins) == 0 { t.Fatalf("the plugin was not attached") } live := newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...) live.Annotations = attached.Annotations f.observeCNPG(live) var backups []cnpgtypes.Backup for range 3 { if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { break } } if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the attach: %+v", backups) } if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) } }) } } // A record older than the latch is left from an earlier attach of a plugin // since removed; the plugin found now was attached by someone who keeps no // record, after the latch, so an instance recreated between the record and // the latch still has to roll. func TestReconcileCNPG_AttachRecordOlderThanTheLatchKeepsTheLatch(t *testing.T) { latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) oldPod := newRolloutInstancePod(latchedAt.Add(-2 * time.Hour)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-2*time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), oldPod) j, _ := f.pass() // latch latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) if latch == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } latch.LastTransitionTime = metav1.NewTime(latchedAt) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the latch: %v", err) } f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(-time.Minute))) live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: latchedAt.Add(-time.Hour).UTC().Format(time.RFC3339)} f.observeCNPG(live) var backups []cnpgtypes.Backup for range 3 { if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { break } } if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the latch: %+v", backups) } if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) } } func (f *cnpgRolloutFixture) liveObjectStore() *cnpgtypes.ObjectStore { f.t.Helper() store := &cnpgtypes.ObjectStore{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil { f.t.Fatalf("get ObjectStore: %v", err) } return store } // setDestinationStamp rewrites the destination-change record on the live // ObjectStore, as a write that long ago, or a corrupt one, would have left it. func (f *cnpgRolloutFixture) setDestinationStamp(value string) { f.t.Helper() store := f.liveObjectStore() if store.Annotations == nil { store.Annotations = map[string]string{} } store.Annotations[cnpgDestinationChangedAtAnnotation] = value if err := f.c.Update(context.Background(), store); err != nil { f.t.Fatalf("stamp ObjectStore: %v", err) } } // settleDestination ages the destination-change record, when the ObjectStore // carries one, as if the plugin's cache had expired since that write. func (f *cnpgRolloutFixture) settleDestination() { f.t.Helper() if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok { f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)) } } // addJobWithDestination adds BackupJob name of the same application, resolved // to a strategy of its own that writes destination, and returns a function // running one pass of it. func (f *cnpgRolloutFixture) addJobWithDestination(name, destination string) func() *backupsv1alpha1.BackupJob { f.t.Helper() ctx := context.Background() strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy-" + name}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: destination}, }}, } if err := f.c.Create(ctx, strategy); err != nil { f.t.Fatalf("seed strategy: %v", err) } job := newRolloutBackupJob(time.Now().Add(-time.Hour)) job.Name = name if err := f.c.Create(ctx, job); err != nil { f.t.Fatalf("seed BackupJob %s: %v", name, err) } group := strategyv1alpha1.GroupVersion.Group resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &group, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}} return func() *backupsv1alpha1.BackupJob { f.t.Helper() key := client.ObjectKey{Namespace: rolloutNS, Name: name} j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(ctx, key, j); err != nil { f.t.Fatalf("get BackupJob %s: %v", name, err) } if _, err := f.r.reconcileCNPG(ctx, j, resolved); err != nil { f.t.Fatalf("reconcileCNPG %s: %v", name, err) } if err := f.c.Get(ctx, key, j); err != nil { f.t.Fatalf("get BackupJob %s after reconcile: %v", name, err) } return j } } // Two jobs of the application with different destinations, neither with a // Backup yet, would otherwise take turns rewriting the ObjectStore, each // write restarting the other's wait for the plugin cache so that neither // ever creates one. The job whose write came first holds the destination, // by its claim until it has a Backup and by that Backup afterwards, and the // other waits for it. func TestReconcileCNPG_JobsWithDifferentDestinationsTakeTurns(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://third/") pass2 := f.addJobWithDestination("bj2", "s3://other/") var j2 *backupsv1alpha1.BackupJob for range 5 { f.pass() f.settleDestination() j2 = pass2() f.settleDestination() } backups := &cnpgtypes.BackupList{} if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil { t.Fatalf("list cnpg.io/Backups: %v", err) } if len(backups.Items) != 1 || backups.Items[0].Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" { t.Fatalf("cnpg.io/Backups = %+v, want exactly one, of bj, whose write came first", backups.Items) } if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, want bj's s3://bucket/ under its running Backup", got) } if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") { t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready) } } // The claim names the job whose write last reached the ObjectStore, whether // it changed the configuration, created the object or left it as it was; the // change record is re-sent as it is by a write that changes nothing. func TestReconcileCNPG_ClaimNamesTheJobThatWroteTheDestination(t *testing.T) { t.Run("changed", func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.pass() if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got) } }) t.Run("created", func(t *testing.T) { f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(time.Now().Add(-time.Hour))) f.pass() store := f.liveObjectStore() if got := store.Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got) } if _, ok := store.Annotations[cnpgDestinationChangedAtAnnotation]; ok { t.Errorf("creating the ObjectStore stamped %s, which would hold the Backup for nothing", cnpgDestinationChangedAtAnnotation) } }) for name, recorded := range map[string]map[string]string{ "unchanged": {cnpgDestinationWrittenByAnnotation: "bj-old"}, "unchanged with a stamp": {cnpgDestinationWrittenByAnnotation: "bj-old", cnpgDestinationChangedAtAnnotation: "2026-01-02T03:04:05Z"}, } { t.Run(name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") store := f.liveObjectStore() store.Annotations = recorded if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed claim: %v", err) } var sent []map[string]string f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { po := &client.PatchOptions{} po.ApplyOptions(opts) if _, ok := obj.(*cnpgtypes.ObjectStore); ok && !slices.Contains(po.DryRun, metav1.DryRunAll) { sent = append(sent, obj.GetAnnotations()) } return cl.Patch(ctx, obj, patch, opts...) }, }) f.pass() if len(sent) != 1 { t.Fatalf("got %d ObjectStore writes, want 1", len(sent)) } if got := sent[0][cnpgDestinationWrittenByAnnotation]; got != "bj" { t.Errorf("the write sent %s = %q, want bj, the job that wrote", cnpgDestinationWrittenByAnnotation, got) } if got, want := sent[0][cnpgDestinationChangedAtAnnotation], recorded[cnpgDestinationChangedAtAnnotation]; got != want { t.Errorf("the write sent %s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, want) } }) } } // A claim stops holding the destination once its job has finished or is // gone, so the next job writes its own. func TestReconcileCNPG_ClaimOfAFinishedJobDoesNotHold(t *testing.T) { for name, phase := range map[string]backupsv1alpha1.BackupJobPhase{ "succeeded": backupsv1alpha1.BackupJobPhaseSucceeded, "failed": backupsv1alpha1.BackupJobPhaseFailed, "gone": "", } { t.Run(name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") if name != "gone" { f.seedOtherBackupJob("bj-other", phase) } store := f.liveObjectStore() store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed claim: %v", err) } j, _ := f.pass() if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, want bj's s3://bucket/", got) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { t.Errorf("Ready = %+v, held by a %s job", ready, name) } }) } } // A job that attaches the plugin writes a configuration the ObjectStore may // already carry, for instance after a restore re-rendered the Cluster // without the plugin. Its write still claims the destination, so a job with // another one waits through the rollout instead of switching it. func TestReconcileCNPG_UnchangedWriteClaimsTheDestinationForTheRollout(t *testing.T) { f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded) store := &cnpgtypes.ObjectStore{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster, Annotations: map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"}}, Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: "s3://bucket/"}}, } if err := f.c.Create(context.Background(), store); err != nil { t.Fatalf("seed ObjectStore: %v", err) } pass2 := f.addJobWithDestination("bj2", "s3://other/") f.pass() // latch if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForPluginRollout" { t.Fatalf("Ready = %+v, want bj waiting for the rollout of the plugin it attached", apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } j2 := pass2() if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, switched during bj's rollout", got) } if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") { t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready) } } // The claim covers only the window before its job has a Backup the queue // can see. Once it has one, the Backup in flight holds the destination, for // as long as CloudNativePG runs it, and a Backup that finished does not, even // if its job is stuck before it finishes, so that job cannot hold every later // backup of the application for as long as it lives. func TestReconcileCNPG_ClaimOfAJobWithItsOwnBackupDoesNotHold(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, cnpgBackupPhaseComplete) store := f.liveObjectStore() store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed claim: %v", err) } j, _ := f.pass() if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, held by a job whose Backup already finished", got) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { t.Errorf("Ready = %+v, held by a job whose Backup already finished", ready) } } // A claim whose job cannot be read cannot be taken as released: writing over // it would let two jobs flip the destination again. The error goes back to // the controller for a retry. func TestReconcileCNPG_ClaimHolderLookupErrorIsReturned(t *testing.T) { unavailable := apierrors.NewServiceUnavailable("apiserver unavailable") for name, setup := range map[string]func(f *cnpgRolloutFixture){ "holder job read fails": func(f *cnpgRolloutFixture) { f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if _, ok := obj.(*backupsv1alpha1.BackupJob); ok && key.Name == "bj-other" { return unavailable } return cl.Get(ctx, key, obj, opts...) }, }) }, "holder Backups list fails": func(f *cnpgRolloutFixture) { holders := labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj-other", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS} f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { lo := &client.ListOptions{} lo.ApplyOptions(opts) if _, ok := list.(*cnpgtypes.BackupList); ok && lo.LabelSelector != nil && lo.LabelSelector.Matches(holders) && !lo.LabelSelector.Matches(labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS}) { return unavailable } return cl.List(ctx, list, opts...) }, }) }, } { t.Run(name, func(t *testing.T) { assertClaimHolderLookupErrorReturned(t, setup) }) } } func assertClaimHolderLookupErrorReturned(t *testing.T, setup func(f *cnpgRolloutFixture)) { t.Helper() f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) store := f.liveObjectStore() store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed claim: %v", err) } setup(f) j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { t.Fatalf("reconcileCNPG swallowed the claim holder lookup error") } if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination written although the claim holder could not be read: %q", got) } } // A running job that holds the claim keeps the destination while it has no // Backup yet, for instance while the plugin it attached rolls the instances. func TestReconcileCNPG_ClaimOfARunningJobHolds(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) store := f.liveObjectStore() store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed claim: %v", err) } j, backups, res := f.passResult() if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination = %q, rewritten under bj-other's claim", got) } if len(backups) != 0 { t.Errorf("a cnpg.io/Backup was created under another job's claim: %+v", backups) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { t.Errorf("Ready = %+v, want WaitingForOtherBackup naming bj-other", ready) } if res.RequeueAfter == 0 { t.Errorf("a held job is not requeued: %+v", res) } } // The plugin sidecar reads the ObjectStore through a cache that nothing this // controller writes invalidates, so a Backup created right after the // destination changed can be taken against the previous one. The job holds // the Backup until that cache has expired, and creates it afterwards. func TestReconcileCNPG_BackupWaitsForThePluginCacheAfterADestinationChange(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") j, backups, res := f.passResult() if got := f.liveDestination(); got != "s3://bucket/" { t.Fatalf("destination = %q, want the job's s3://bucket/", got) } if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created in the pass that changed the destination: %+v", backups) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForDestinationToSettle" { t.Errorf("Ready = %+v, want WaitingForDestinationToSettle", ready) } if res.RequeueAfter == 0 { t.Errorf("a waiting job is not requeued: %+v", res) } stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]) if err != nil || time.Since(stamp) > time.Minute { t.Fatalf("%s = %v (%v), want the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err) } if _, backups = f.pass(); len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created while the plugin cache could still hold the old destination: %+v", backups) } f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)) if j, backups = f.pass(); len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups once the cache expired, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } } // A write that leaves the configuration as it was cannot leave a stale // cache behind: the Backup is created at once, and the record of the last // change is re-sent as it is, since an apply that left it out would delete it. func TestReconcileCNPG_UnchangedDestinationDoesNotWait(t *testing.T) { const stamp = "2026-01-02T03:04:05Z" f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") f.setDestinationStamp(stamp) if j, backups := f.pass(); len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups for an unchanged destination, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } if got := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; got != stamp { t.Errorf("%s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, stamp) } } // The apiserver answers an apply with the merged object: fields it defaults, // or that another field manager owns, are there although the job never sets // them. Judged against what the job sends, every pass would read as a change // and the Backup would never be created; judged against the dry run of the // same apply, an unchanged configuration stays unchanged. func TestReconcileCNPG_FieldsTheJobDoesNotSetAreNotAChange(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") store := f.liveObjectStore() store.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"} if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed ObjectStore: %v", err) } f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { po := &client.PatchOptions{} po.ApplyOptions(opts) if os, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) { os.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"} return nil } return cl.Patch(ctx, obj, patch, opts...) }, }) if j, backups := f.pass(); len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups for a configuration only the server filled in, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok { t.Errorf("%s was set by a write that changed nothing", cnpgDestinationChangedAtAnnotation) } } // Without the live ObjectStore, or the dry run of the apply, the job cannot // tell whether its write changes the destination, and taking the write as // unchanged would create the Backup at once against a cache that may still // hold the previous one. Either failure is retried like a failed write. func TestReconcileCNPG_UnreadableObjectStoreIsRetried(t *testing.T) { unavailable := apierrors.NewServiceUnavailable("apiserver unavailable") for _, tc := range []struct { name string setup func(f *cnpgRolloutFixture) }{ {name: "live read fails", setup: func(f *cnpgRolloutFixture) { f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok { return unavailable } return cl.Get(ctx, key, obj, opts...) }, }) }}, {name: "claim read fails", setup: func(f *cnpgRolloutFixture) { reads := 0 f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok { if reads++; reads == 1 { return unavailable } } return cl.Get(ctx, key, obj, opts...) }, }) }}, {name: "live read at the write fails", setup: func(f *cnpgRolloutFixture) { reads := 0 f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok { if reads++; reads > 1 { return unavailable } } return cl.Get(ctx, key, obj, opts...) }, }) }}, {name: "dry run fails", setup: func(f *cnpgRolloutFixture) { f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { po := &client.PatchOptions{} po.ApplyOptions(opts) if _, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) { return unavailable } return cl.Patch(ctx, obj, patch, opts...) }, }) }}, } { t.Run(tc.name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") tc.setup(f) assertObjectStoreReadRetried(t, f) }) } } // assertObjectStoreReadRetried runs one pass and checks it is retried, either // by returning the error to the controller or by waiting with // ClusterPluginApplyRetrying, without a Backup, a destination write or a // failed job. func assertObjectStoreReadRetried(t *testing.T, f *cnpgRolloutFixture) { t.Helper() ctx := context.Background() key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"} j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(ctx, key, j); err != nil { t.Fatalf("get BackupJob: %v", err) } res, reconcileErr := f.r.reconcileCNPG(ctx, j, f.resolved) if err := f.c.Get(ctx, key, j); err != nil { t.Fatalf("get BackupJob after reconcile: %v", err) } backups := &cnpgtypes.BackupList{} if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil { t.Fatalf("list cnpg.io/Backups: %v", err) } if len(backups.Items) != 0 { t.Fatalf("a cnpg.io/Backup was created although the ObjectStore could not be read: %+v", backups.Items) } if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination = %q, written without knowing whether it changed", got) } if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the job failed on a transient read: %s", j.Status.Message) } if reconcileErr != nil { return } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "ClusterPluginApplyRetrying" { t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying", ready) } if res.RequeueAfter == 0 { t.Errorf("a retried read is not requeued: %+v", res) } } // A change record that cannot be read, or that lies further ahead than // controller clocks drift, says nothing about when the cache expires, so the // write replaces it with its own time and the job waits once. It claims the // destination for that wait, or another job would write its own meanwhile. func TestReconcileCNPG_UnusableDestinationRecordIsReplaced(t *testing.T) { for name, value := range map[string]string{ "does not parse": "yesterday", "far ahead": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), } { t.Run(name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded) store := f.liveObjectStore() store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"} if err := f.c.Update(context.Background(), store); err != nil { t.Fatalf("seed claim: %v", err) } f.setDestinationStamp(value) j, backups := f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created with an unusable change record: %+v", backups) } if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the job failed: %s", j.Status.Message) } stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]) if err != nil || time.Since(stamp) > time.Minute || time.Until(stamp) > time.Minute { t.Fatalf("%s = %v (%v), want it replaced with the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err) } if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { t.Errorf("%s = %q, want bj, which waits on the replaced record", cnpgDestinationWrittenByAnnotation, got) } }) } } // A later write to a Cluster that already carries the plugin re-sends the // attach record as it is: moving it would make the instances that rolled // with the plugin look older than the attach. func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) { const stamp = "2026-01-02T03:04:05Z" cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, newRolloutInstancePod(time.Now().Add(-time.Hour))) var sent []map[string]string f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.Cluster); ok { sent = append(sent, obj.GetAnnotations()) } return cl.Patch(ctx, obj, patch, opts...) }, }) if _, backups := f.pass(); len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups, want the one this pass creates", len(backups)) } if len(sent) != 1 { t.Fatalf("got %d Cluster writes, want 1", len(sent)) } if got := sent[0][cnpgPluginAttachedAtAnnotation]; got != stamp { t.Errorf("the write sent %s = %q, want the recorded %q", cnpgPluginAttachedAtAnnotation, got, stamp) } } // An attach record that does not parse cannot tell which instances rolled // with the plugin, so the job fails naming it instead of guessing. One further // ahead than controller clocks drift apart would hold every instance as older // than the plugin and push the rollout deadline out by as far as it lies // ahead, so it fails the same way. func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) { for _, tc := range []struct{ name, record string }{ {name: "does not parse", record: "yesterday"}, {name: "in the future", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339)}, } { t.Run(tc.name, func(t *testing.T) { f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} f.observeCNPG(live) j, backups := f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created with an unreadable attach record: %+v", backups) } if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgPluginAttachedAtAnnotation) { t.Errorf("phase %q, message %q; want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgPluginAttachedAtAnnotation) } }) } } // A record a little ahead of this controller's clock is what a leadership // change between controllers whose clocks differ leaves behind, so the job // waits for the clocks to agree rather than failing an attach that worked. func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) { f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)} f.observeCNPG(live) j, backups, res := f.passResult() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the job failed on a record a minute ahead: %s", j.Status.Message) } if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created before the rollout could be judged: %+v", backups) } if res.RequeueAfter == 0 { t.Errorf("a waiting job is not requeued: %+v", res) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) } } // A job that attached nothing itself still finds the plugin another backup // job attached a moment ago, while CNPG rolls the instances for it; a Backup // created then fails for good. It waits for that rollout as long as the // attach is recent enough for it to be running, and an attach record it // cannot read is not its own to fail on. func TestReconcileCNPG_JobWithoutLatchWaitsForAnotherJobsRollout(t *testing.T) { cases := []struct { name string record string podCreated time.Time wantBackup bool }{ {name: "recent attach, instance not rolled yet", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-10 * time.Minute), wantBackup: false}, {name: "recent attach, instance rolled", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Minute), wantBackup: true}, {name: "attach older than the rollout deadline", record: time.Now().Add(-2 * cnpgDefaultBackupDeadline).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-3 * cnpgDefaultBackupDeadline), wantBackup: true}, {name: "unreadable record", record: "yesterday", podCreated: time.Now().Add(-time.Hour), wantBackup: true}, {name: "record further ahead than clocks drift", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: true}, {name: "record ahead within clock drift", record: time.Now().Add(time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, newRolloutInstancePod(tc.podCreated.Truncate(time.Second))) j, backups, res := f.passResult() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the job failed: %s", j.Status.Message) } if got := len(backups) == 1; got != tc.wantBackup { t.Fatalf("cnpg.io/Backup created = %t, want %t; Ready = %+v", got, tc.wantBackup, apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } if !tc.wantBackup { if res.RequeueAfter == 0 { t.Errorf("a waiting job is not requeued: %+v", res) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) } } }) } } // A job whose own Backup already exists is past every wait before it: it // tracks that Backup even while another job's recent attach is still rolling // the instances, since waiting would only hold the result of a Backup that // runs regardless. func TestReconcileCNPG_JobWithItsOwnBackupDoesNotWaitForAnotherJobsRollout(t *testing.T) { cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339)} f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, newRolloutInstancePod(time.Now().Add(-10*time.Minute).Truncate(time.Second))) f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") j, _ := f.pass() if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { t.Errorf("Ready = %+v, want WaitingForCNPGBackup tracking the job's own Backup", ready) } } // A plugin attached by someone else after the latch, another backup job or // the chart re-rendering a restored Cluster outside the platform flow, // started the rollout before this job's write, which only finds it there: // instances that rolled since the latch carry the plugin. func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) { oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), newRolloutCluster(cnpgClusterHealthyPhase), oldPod) j, _ := f.pass() // latch latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) if latch == nil { t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) } latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) latch.LastTransitionTime = metav1.NewTime(latchedAt) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the latch: %v", err) } f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))) f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(5*time.Minute))) var backups []cnpgtypes.Backup for range 3 { if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { break } } if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups against instances that rolled with the plugin; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } } // Waiting for another backup resets the retry budget of the destination // write, so the queue does not spend it. func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) f := newAttachedObjectStoreFixture(t, expired, "s3://other/") f.setJobCondition(metav1.Condition{ Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), }) f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") j, _ := f.pass() if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond) } held := &cnpgtypes.Backup{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { t.Fatalf("get holder Backup: %v", err) } held.Status.Phase = cnpgBackupPhaseComplete if err := f.c.Update(context.Background(), held); err != nil { t.Fatalf("complete holder Backup: %v", err) } f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok { return apierrors.NewInternalError(errors.New("webhook down")) } return cl.Patch(ctx, obj, patch, opts...) }, }) j, _ = f.pass() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the first failure after the queue failed the job: %s", j.Status.Message) } } // seedRestoreOfApp starts a RestoreJob that writes into the fixture's app. func (f *cnpgRolloutFixture) seedRestoreOfApp() *backupsv1alpha1.RestoreJob { f.t.Helper() appGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-backup"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &appGroup, Kind: postgresAppKind, Name: rolloutApp}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, }, } restore := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-restore"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, } for _, obj := range []client.Object{backup, restore} { if err := f.c.Create(context.Background(), obj); err != nil { f.t.Fatalf("seed %T: %v", obj, err) } } return restore } // The restore check before the destination write reads the apiserver: the // cache can still miss a RestoreJob created a moment ago, and a write landing // in that gap can put the restored Cluster back on the source's WAL prefix. func TestReconcileCNPG_RestoreCheckReadsPastTheCache(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.seedRestoreOfApp() f.r.APIReader = f.c f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { return nil } return cl.List(ctx, list, opts...) }, }) j, backups := f.pass() if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination written during a restore the cache had not caught: %q", got) } if len(backups) != 0 { t.Errorf("a cnpg.io/Backup was created during a restore the cache had not caught: %+v", backups) } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress { t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress) } } // A job past its start writes the WAL archive destination only while no // restore rewrites its application. The restore gate in Reconcile looks only // at a job that has not started, and a job released from the queue, or one // still before its first write, has started long ago: a write landing during // the restore can put the restored Cluster back on the source's WAL prefix. func TestReconcileCNPG_HoldsTheDestinationWriteWhileTheAppIsRestored(t *testing.T) { for _, queued := range []bool{true, false} { t.Run(fmt.Sprintf("released from the queue=%t", queued), func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") if queued { f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForOtherBackup" { t.Fatalf("Ready = %+v, want the job queued", apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } } restore := f.seedRestoreOfApp() if queued { held := &cnpgtypes.Backup{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { t.Fatalf("get holder Backup: %v", err) } held.Status.Phase = cnpgBackupPhaseComplete if err := f.c.Update(context.Background(), held); err != nil { t.Fatalf("complete holder Backup: %v", err) } holder := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other"}, holder); err != nil { t.Fatalf("get holder BackupJob: %v", err) } holder.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded if err := f.c.Status().Update(context.Background(), holder); err != nil { t.Fatalf("finish holder BackupJob: %v", err) } } j, backups, res := f.passResult() if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination written during a restore of the app: %q", got) } for _, b := range backups { if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { t.Errorf("a cnpg.io/Backup was created during a restore of the app: %s", b.Name) } } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress { t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress) } if res.RequeueAfter == 0 { t.Errorf("a held job is not requeued: %+v", res) } restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded if err := f.c.Status().Update(context.Background(), restore); err != nil { t.Fatalf("finish the restore: %v", err) } f.pass() if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("once the restore finished, destination = %q, want the job's own s3://bucket/", got) } }) } } // A job whose Backup already runs writes nothing, so a restore does not hold // it: holding would strand a backup that is already moving data. func TestReconcileCNPG_RestoreDoesNotHoldARunningBackup(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") f.seedRestoreOfApp() j, _ := f.pass() if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { t.Errorf("Ready = %+v, want the running backup still followed", ready) } } // An error looking for a restore is not taken as "no restore". func TestReconcileCNPG_RestoreLookupErrorIsReturned(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { return apierrors.NewServiceUnavailable("cache not synced") } return cl.List(ctx, list, opts...) }, }) j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { t.Fatalf("reconcileCNPG swallowed the restore lookup error") } if got := f.liveDestination(); got != "s3://other/" { t.Errorf("destination written although restores could not be listed: %q", got) } } // Waiting for a restore resets the retry budget of the destination write, // as waiting for another backup does, so the restore does not spend it. func TestReconcileCNPG_RestoreResetsDestinationRetryBudget(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) f := newAttachedObjectStoreFixture(t, expired, "s3://other/") f.setJobCondition(metav1.Condition{ Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), }) restore := f.seedRestoreOfApp() j, _ := f.pass() if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { t.Fatalf("%s = %+v kept while waiting for a restore", backupCondDestinationApplied, cond) } restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded if err := f.c.Status().Update(context.Background(), restore); err != nil { t.Fatalf("finish the restore: %v", err) } f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*cnpgtypes.ObjectStore); ok { return apierrors.NewInternalError(errors.New("webhook down")) } return cl.Patch(ctx, obj, patch, opts...) }, }) if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("the first failure after the restore failed the job: %s", j.Status.Message) } } // With two jobs holding the queue the wait names the same one on every pass, // whatever order the cache lists their Backups in, so a waiting job does not // rewrite its status each poll. func TestReconcileCNPG_QueueNamesOneHolderAcrossPasses(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedOtherBackupJob("holder-z", backupsv1alpha1.BackupJobPhaseRunning) f.seedOtherBackupJob("holder-a", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("a-cnpg", "holder-z", rolloutCluster, "running") f.seedCNPGBackup("z-cnpg", "holder-a", rolloutCluster, "running") statusWrites := 0 f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { statusWrites++ return cl.SubResource(sub).Update(ctx, obj, opts...) }, }) j, _ := f.pass() if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "holder-a ") { t.Fatalf("Ready = %+v, want WaitingForOtherBackup naming holder-a", ready) } statusWrites = 0 f.pass() if statusWrites != 0 { t.Errorf("a second pass with the same two holders wrote the BackupJob status %d time(s)", statusWrites) } } func TestOtherCNPGBackupInFlight_IgnoresTheJobsOwnBackup(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } holder, err := f.r.otherCNPGBackupInFlight(context.Background(), j, rolloutCluster) if err != nil { t.Fatalf("otherCNPGBackupInFlight: %v", err) } if holder != "" { t.Fatalf("the job waits on its own Backup: holder %q", holder) } } // Once the job's Backup exists the Cluster is only read; a failed read is // returned for a retry and neither fails the running backup nor starts the // destination retry budget. func TestReconcileCNPG_ClusterReadErrorDoesNotFailRunningBackup(t *testing.T) { for name, readErr := range map[string]error{ "internal error": apierrors.NewInternalError(errors.New("blip")), // A write answered Forbidden fails the job; a read must not. "forbidden": apierrors.NewForbidden(schema.GroupResource{Group: cnpgtypes.GroupName, Resource: "clusters"}, rolloutCluster, errors.New("denied")), } { t.Run(name, func(t *testing.T) { testClusterReadErrorKeepsBackup(t, readErr) }) } } func testClusterReadErrorKeepsBackup(t *testing.T, readErr error) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) f := newAttachedObjectStoreFixture(t, expired, "s3://other/") f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") f.setJobCondition(metav1.Condition{ Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), }) dyn := f.r.Interface.(*dynamicfake.FakeDynamicClient) dyn.PrependReactor("get", "clusters", func(clienttesting.Action) (bool, runtime.Object, error) { return true, nil, readErr }) j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { t.Fatalf("a failed Cluster read was not returned for a retry") } persisted := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { t.Fatalf("get BackupJob: %v", err) } if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("a failed Cluster read failed the running backup: %s", persisted.Status.Message) } } // cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose // Cluster and instance pods a test swaps between passes, standing in for // CNPG rolling the instances. type cnpgRolloutFixture struct { t *testing.T c client.Client r *BackupJobReconciler resolved *ResolvedBackupConfig } const ( rolloutNS = "tenant" rolloutApp = "app" rolloutCluster = "postgres-app" ) func newCNPGRolloutFixture(t *testing.T, j *backupsv1alpha1.BackupJob, cluster *cnpgtypes.Cluster, pods ...*corev1.Pod) *cnpgRolloutFixture { t.Helper() strategyGroup := strategyv1alpha1.GroupVersion.Group strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } objs := []client.Object{j, strategy, newPostgresApp(rolloutApp, rolloutNS), cluster} for _, p := range pods { objs = append(objs, p) } c := newCNPGStrategyTestClient(t, objs...) return &cnpgRolloutFixture{ t: t, c: c, r: &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)}, resolved: &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name, }}, } } func newRolloutBackupJob(startedAt time.Time) *backupsv1alpha1.BackupJob { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup started := metav1.NewTime(startedAt) return &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: rolloutApp}, }, Status: backupsv1alpha1.BackupJobStatus{StartedAt: &started}, } } func newRolloutCluster(phase string, plugins ...cnpgtypes.PluginConfiguration) *cnpgtypes.Cluster { return &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster, UID: "cluster-uid"}, Spec: cnpgtypes.ClusterSpec{Plugins: plugins}, Status: cnpgtypes.ClusterStatus{Phase: phase}, } } func newRolloutInstancePod(created time.Time) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: rolloutNS, Name: rolloutCluster + "-1", CreationTimestamp: metav1.NewTime(created), Labels: map[string]string{cnpgClusterLabel: rolloutCluster, cnpgPodRoleLabel: cnpgPodRoleInstance}, }, Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}, } } // pass runs one reconcile and returns the persisted BackupJob and the // cnpg.io/Backups that exist afterwards. func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup) { f.t.Helper() j, backups, _ := f.passResult() return j, backups } // passResult is pass that also returns the reconcile result: the controller // watches only BackupJobs, so a pass that waits on anything else is woken // only by its own requeue. func (f *cnpgRolloutFixture) passResult() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup, ctrl.Result) { f.t.Helper() ctx := context.Background() key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"} j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(ctx, key, j); err != nil { f.t.Fatalf("get BackupJob: %v", err) } res, err := f.r.reconcileCNPG(ctx, j, f.resolved) if err != nil { f.t.Fatalf("reconcileCNPG: %v", err) } got := &backupsv1alpha1.BackupJob{} if err := f.c.Get(ctx, key, got); err != nil { f.t.Fatalf("get BackupJob after reconcile: %v", err) } backups := &cnpgtypes.BackupList{} if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil { f.t.Fatalf("list cnpg.io/Backups: %v", err) } return got, backups.Items, res } // observeCNPG makes the next live read see cluster, as CNPG has written it. func (f *cnpgRolloutFixture) observeCNPG(cluster *cnpgtypes.Cluster) { f.r.Interface = cnpgDynamicFor(f.t, cluster) } func (f *cnpgRolloutFixture) replacePod(old, recreated *corev1.Pod) { f.t.Helper() if err := f.c.Delete(context.Background(), old); err != nil { f.t.Fatalf("delete pod: %v", err) } if err := f.c.Create(context.Background(), recreated); err != nil { f.t.Fatalf("create pod: %v", err) } } // TestReconcileCNPG_WaitsForPluginRolloutBeforeBackup covers the first backup // of a Cluster that had no barman-cloud plugin. Attaching it makes CNPG roll // every instance, and a cnpg.io/Backup created before the rollout is over // fails for good ("requested plugin is not available"). No Backup may exist // until the instances have been recreated and CNPG reports the Cluster // healthy again, including in the window where CNPG has not reconciled the // new spec yet and the phase still reads healthy from before. func TestReconcileCNPG_WaitsForPluginRolloutBeforeBackup(t *testing.T) { oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second)) f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)), newRolloutCluster(cnpgClusterHealthyPhase), oldPod) j, backups := f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created before the plugin was even attached: %+v", backups) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); cond == nil || cond.Status != metav1.ConditionFalse { t.Fatalf("%s condition = %+v, want False before the plugin is attached", backupCondPluginRolledOut, cond) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" { t.Fatalf("Ready condition = %+v, want reason WaitingForPluginRollout", cond) } attachedAt := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut).LastTransitionTime _, backups = f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created in the pass that attached the plugin: %+v", backups) } attached := &cnpgtypes.Cluster{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { t.Fatalf("get Cluster: %v", err) } if currentBarmanServerName(attached) == "" { t.Fatalf("the barman-cloud plugin was never attached: %+v", attached.Spec.Plugins) } // CNPG has not picked the new spec up yet: its phase is the one from // before the attach. f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) j, backups = f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created on a stale healthy phase, against instances without the plugin: %+v", backups) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || !strings.Contains(cond.Message, "predates") { t.Fatalf("Ready condition = %+v, want the instance pod that predates the plugin in its message", cond) } rolling := newRolloutCluster("Upgrading cluster", attached.Spec.Plugins...) rolling.Status.PhaseReason = "Restarting instance postgres-app-1, because: volumes: element plugins has been added" f.observeCNPG(rolling) recreated := newRolloutInstancePod(attachedAt.Add(time.Minute)) f.replacePod(oldPod, recreated) j, backups = f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created while CNPG was rolling the instances: %+v", backups) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || !strings.Contains(cond.Message, "Upgrading cluster") { t.Fatalf("Ready condition = %+v, want the rolling phase in its message", cond) } f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) j, backups = f.pass() if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups once the rollout is over, want 1", len(backups)) } if j.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning { t.Errorf("BackupJob phase = %q, want Running", j.Status.Phase) } if !apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) { t.Errorf("%s condition = %+v, want True", backupCondPluginRolledOut, apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForCNPGBackup" { t.Errorf("Ready condition = %+v, want the wait on the cnpg.io/Backup rather than one left over from the rollout", cond) } } // TestReconcileCNPG_AttachedPluginDoesNotDelayBackup: a Cluster that already // runs the plugin is not rolled by the driver, so its Backup is created on the // first pass, whatever the age of its pods. func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) { f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)), newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(time.Now().Add(-time.Hour))) j, backups := f.pass() if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups on the first pass, want 1", len(backups)) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); cond != nil { t.Errorf("%s condition = %+v on a Cluster the driver did not roll", backupCondPluginRolledOut, cond) } if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); cond != nil { t.Errorf("%s condition = %+v on a job nothing held", backupCondDestinationWriteHeld, cond) } } // An in-place restore of the platform flow recreates the Cluster without the // barman-cloud plugin, so a job held through it attaches the plugin again // once released. That attach starts a rollout of its own, which the // instances restored after the job's first rollout began have not done. func TestReconcileCNPG_ReattachAfterRestoreRestartsTheRollout(t *testing.T) { stampedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) j.Status.Conditions = []metav1.Condition{{ Type: backupCondPluginRolledOut, Status: metav1.ConditionFalse, Reason: "WaitingForPluginRollout", LastTransitionTime: metav1.NewTime(stampedAt), }} f := newCNPGRolloutFixture(t, j, newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-2*time.Minute))) restore := f.seedRestoreOfApp() if got, _ := f.pass(); apimeta.FindStatusCondition(got.Status.Conditions, "Ready").Reason != ConditionReasonRestoreInProgress { t.Fatalf("Ready = %+v, want the job held by the restore", apimeta.FindStatusCondition(got.Status.Conditions, "Ready")) } restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded if err := f.c.Status().Update(context.Background(), restore); err != nil { t.Fatalf("finish the restore: %v", err) } attachAt := time.Now().Truncate(time.Second) got, backups := f.pass() if len(backups) != 0 { t.Fatalf("cnpg.io/Backup created in the pass that attached the plugin to the restored Cluster: %+v", backups) } rollout := apimeta.FindStatusCondition(got.Status.Conditions, backupCondPluginRolledOut) if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) } } // A release older than the rollout, from a wait before the plugin was // attached, does not shorten the rollout that began after it. func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) j := newRolloutBackupJob(expired.Add(-time.Hour)) j.Status.Conditions = []metav1.Condition{ { Type: backupCondDestinationWriteHeld, Status: metav1.ConditionFalse, Reason: "Released", LastTransitionTime: metav1.NewTime(expired), }, { Type: backupCondPluginRolledOut, Status: metav1.ConditionFalse, Reason: "WaitingForPluginRollout", LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)), }, } f := newCNPGRolloutFixture(t, j, newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(expired.Add(-time.Hour))) got, backups := f.pass() if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("a rollout that began a minute ago failed on an older release: %s", got.Status.Message) } if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created against an unfinished rollout: %+v", backups) } } // TestReconcileCNPG_PluginRolloutWaitIsBounded: a rollout that never ends // fails the BackupJob at the backup deadline instead of holding it Running. func TestReconcileCNPG_PluginRolloutWaitIsBounded(t *testing.T) { startedAt := time.Now().Add(-cnpgDefaultBackupDeadline - time.Minute) j := newRolloutBackupJob(startedAt) j.Status.Conditions = []metav1.Condition{{ Type: backupCondPluginRolledOut, Status: metav1.ConditionFalse, Reason: "WaitingForPluginRollout", LastTransitionTime: metav1.NewTime(startedAt), }} f := newCNPGRolloutFixture(t, j, newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(startedAt.Add(-time.Hour))) got, backups := f.pass() if len(backups) != 0 { t.Fatalf("a cnpg.io/Backup was created against an unfinished rollout: %+v", backups) } if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("BackupJob phase = %q, want Failed past the deadline", got.Status.Phase) } if !strings.Contains(got.Status.Message, "Waiting for user action") { t.Errorf("BackupJob message = %q, want the phase CNPG is stuck in", got.Status.Message) } } // A job whose plugin rollout started, and which then waited for another // backup of the app or for a restore of it, gets the rollout deadline back // from the moment it was released: that wait is not a rollout of its own. // The instance pods still count as rolled from the rollout's own start. func TestReconcileCNPG_HeldJobRolloutDeadlineCountsFromRelease(t *testing.T) { type hold struct { name string seed func(f *cnpgRolloutFixture) (release func()) waiting string } holds := []hold{ { name: "another backup", waiting: "WaitingForOtherBackup", seed: func(f *cnpgRolloutFixture) func() { f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") return func() { held := &cnpgtypes.Backup{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { f.t.Fatalf("get holder Backup: %v", err) } held.Status.Phase = cnpgBackupPhaseComplete if err := f.c.Update(context.Background(), held); err != nil { f.t.Fatalf("complete holder Backup: %v", err) } } }, }, { name: "a restore", waiting: ConditionReasonRestoreInProgress, seed: func(f *cnpgRolloutFixture) func() { restore := f.seedRestoreOfApp() return func() { restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded if err := f.c.Status().Update(context.Background(), restore); err != nil { f.t.Fatalf("finish the restore: %v", err) } } }, }, } stampedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) newHeldRollout := func(t *testing.T, h hold, cluster *cnpgtypes.Cluster, pod *corev1.Pod) *cnpgRolloutFixture { t.Helper() j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) j.Status.Conditions = []metav1.Condition{{ Type: backupCondPluginRolledOut, Status: metav1.ConditionFalse, Reason: "WaitingForPluginRollout", LastTransitionTime: metav1.NewTime(stampedAt), }} f := newCNPGRolloutFixture(t, j, cluster, pod) release := h.seed(f) got, backups := f.pass() for _, b := range backups { if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { t.Fatalf("a held job created cnpg.io/Backup %s", b.Name) } } if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("a held job did not wait: %s", got.Status.Message) } if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready == nil || ready.Reason != h.waiting { t.Fatalf("Ready = %+v, want %s", ready, h.waiting) } release() return f } for _, h := range holds { t.Run(h.name+": the rollout is still waited for after the release", func(t *testing.T) { f := newHeldRollout(t, h, newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(stampedAt.Add(-time.Hour))) releasedAt := time.Now().Truncate(time.Second) j, backups := f.pass() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("time held by %s was charged to the rollout: %s", h.name, j.Status.Message) } for _, b := range backups { if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { t.Fatalf("cnpg.io/Backup %s was created against an unfinished rollout", b.Name) } } if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { t.Fatalf("Ready = %+v, want WaitingForPluginRollout", ready) } released := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) if released == nil || released.Status != metav1.ConditionFalse || released.LastTransitionTime.Time.Before(releasedAt) { t.Fatalf("%s = %+v, want False since the release at %s", backupCondDestinationWriteHeld, released, releasedAt) } if rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); rollout == nil || !rollout.LastTransitionTime.Time.Equal(stampedAt) { t.Errorf("%s = %+v, want its stamp kept at %s", backupCondPluginRolledOut, rollout, stampedAt) } released.LastTransitionTime = metav1.NewTime(stampedAt) if err := f.c.Status().Update(context.Background(), j); err != nil { t.Fatalf("age the release: %v", err) } j, _ = f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { t.Fatalf("phase = %q (%s), want Failed once the rollout outlived the deadline after the release", j.Status.Phase, j.Status.Message) } }) t.Run(h.name+": an instance that rolled while the job was held counts", func(t *testing.T) { f := newHeldRollout(t, h, newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), newRolloutInstancePod(stampedAt.Add(time.Minute))) var j *backupsv1alpha1.BackupJob own := 0 for range 3 { var backups []cnpgtypes.Backup j, backups = f.pass() own = 0 for _, b := range backups { if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { own++ } } if own != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { break } } if own != 1 { t.Fatalf("got %d cnpg.io/Backups of the job against an instance that rolled after the rollout began; Ready = %+v, phase %q (%s)", own, apimeta.FindStatusCondition(j.Status.Conditions, "Ready"), j.Status.Phase, j.Status.Message) } }) } } // TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline covers the // restore-path blocker: a RestoreJob whose referenced CNPG Strategy CR never // appears must NOT requeue forever. requeueRestoreStrategyNotReady gates // terminal-vs-transient on Status.StartedAt, and the "fail closed" guarantee // only holds if every driver stamps StartedAt before it can reach the Strategy // NotFound branch. reconcileCNPGRestore stamps StartedAt and returns early on // the first reconcile (well before the strategy lookup), so the deadline clock // starts even when the job begins with StartedAt==nil; once the deadline // elapses the next reconcile flips the RestoreJob to Phase=Failed. // // The existing TestRequeueRestoreStrategyNotReady_BoundedByDeadline only // exercises the helper in isolation with StartedAt pre-populated, so it cannot // catch a driver that reaches the NotFound branch without ever stamping // StartedAt. This test proves the guarantee end to end at the driver call site. func TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "missing-strategy", }, }, } // Status.StartedAt deliberately nil: the driver itself must start the clock. rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, } // No CNPG Strategy CR seeded - the NotFound branch is the point. c := newCNPGStrategyTestClient(t, backup, rj) r := &RestoreJobReconciler{Client: c} ctx := context.Background() // Pass 1: StartedAt is nil, so the driver stamps it and requeues. It must // NOT fail yet - the bootstrap grace window has only just opened. res, err := r.reconcileCNPGRestore(ctx, rj, backup) if err != nil { t.Fatalf("pass 1 reconcileCNPGRestore: %v", err) } if res.RequeueAfter == 0 { t.Fatalf("pass 1 must requeue after stamping StartedAt, got %+v", res) } persisted := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKeyFromObject(rj), persisted); err != nil { t.Fatalf("get after pass 1: %v", err) } if persisted.Status.StartedAt == nil { t.Fatalf("driver did not stamp StartedAt before the Strategy NotFound branch; " + "the deadline clock would never start and the RestoreJob would requeue forever") } if persisted.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("must not fail within the grace window, got Phase=%q", persisted.Status.Phase) } // Simulate StrategyNotReadyDeadline of wall-clock elapsing by backdating the // persisted StartedAt past the deadline. stale := metav1.NewTime(persisted.Status.StartedAt.Add(-StrategyNotReadyDeadline - time.Minute)) base := persisted.DeepCopy() persisted.Status.StartedAt = &stale if err := c.Status().Patch(ctx, persisted, client.MergeFrom(base)); err != nil { t.Fatalf("backdate StartedAt: %v", err) } // Pass 2: StartedAt is now past the deadline and the Strategy CR still does // not exist, so the driver call site must fail closed. reloaded := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKeyFromObject(rj), reloaded); err != nil { t.Fatalf("get before pass 2: %v", err) } res, err = r.reconcileCNPGRestore(ctx, reloaded, backup) if err != nil { t.Fatalf("pass 2 reconcileCNPGRestore: %v", err) } if res.RequeueAfter != 0 { t.Fatalf("terminal failure must not requeue, got %+v", res) } final := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKeyFromObject(rj), final); err != nil { t.Fatalf("get after pass 2: %v", err) } if final.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("expected Phase=Failed once the deadline elapsed, got %q", final.Status.Phase) } if !strings.Contains(final.Status.Message, "missing-strategy") { t.Errorf("failure message should name the missing strategy, got %q", final.Status.Message) } } // TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster is the // reconcile-level regression test for #3311. The pure-function tests // (TestCNPGPurgeNeeded / TestCNPGClusterFreshlyRecovered) lock in the truth // table, but the bug lived at the call site: a repeat in-place restore fed the // purge decision the wrong signal and skipped the destructive purge, so the // job reported Succeeded against untouched data. This drives the real // reconcileCNPGRestore path with a fake client and asserts the destructive // purge actually fires for a stale leftover recovery Cluster - and, in the // mirror case, that a Cluster this restore just re-created is NOT re-purged // (the status-write-race protection the guard was originally built for). func TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() // startedAt anchors the discriminator: a stale leftover Cluster predates // it, a freshly-recovered one postdates it. startedAt := metav1.NewTime(time.Now()) mkBackupArtifact := func(t *testing.T) *backupsv1alpha1.Backup { t.Helper() snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } return &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{ Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }, }, } // A completed cnpg.io/Backup with endWal set clears the WAL-archive gate // so the reconcile can reach the purge step. cnpgBackup := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003"}, } mkRecoveryCluster := func(created metav1.Time) *cnpgtypes.Cluster { return &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: created}, Spec: cnpgtypes.ClusterSpec{ Bootstrap: &cnpgtypes.BootstrapConfiguration{ Recovery: &cnpgtypes.RecoverySource{Source: appName}, }, }, } } mkClusterPVC := func() *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName + "-1", Labels: map[string]string{cnpgClusterLabel: clusterName}}, } } mkRestoreJob := func() *backupsv1alpha1.RestoreJob { sa := startedAt return &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, } } t.Run("stale leftover recovery cluster from a prior restore is purged", func(t *testing.T) { backup := mkBackupArtifact(t) // creationTimestamp an hour before StartedAt: leftover from a prior restore. stale := metav1.NewTime(startedAt.Add(-time.Hour)) c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, newPostgresApp(appName, ns), mkRecoveryCluster(stale), mkClusterPVC()) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } // The stale Cluster (and its labelled PVC) must have been purged. err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}) if !apierrors.IsNotFound(err) { t.Fatalf("expected stale Cluster to be purged (NotFound), got err=%v", err) } pvcs := &corev1.PersistentVolumeClaimList{} if err := c.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{cnpgClusterLabel: clusterName}); err != nil { t.Fatalf("list PVCs: %v", err) } if len(pvcs.Items) != 0 { t.Fatalf("expected cluster PVCs purged, still have %d", len(pvcs.Items)) } }) t.Run("unmarked cluster this restore's values rendered is not re-purged", func(t *testing.T) { backup := mkBackupArtifact(t) // creationTimestamp a minute after StartedAt: this restore's own // re-render by a chart that does not annotate the Cluster. fresh := metav1.NewTime(startedAt.Add(time.Minute)) rjSeed := mkRestoreJob() app := newPostgresApp(appName, ns) app.Spec.Bootstrap.NewServerName = restoredServerName(clusterName, rjSeed.UID) c := newCNPGStrategyTestClient(t, backup, rjSeed, strategy, cnpgBackup, app, mkRecoveryCluster(fresh)) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } // The freshly-recovered Cluster must survive - re-purging it would // destroy the recovery this restore just started (the status-write-race // protection the guard was built for). got := &cnpgtypes.Cluster{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, got); err != nil { t.Fatalf("expected freshly-recovered Cluster to survive, got err=%v", err) } if !got.DeletionTimestamp.IsZero() { t.Fatalf("freshly-recovered Cluster must not be marked for deletion") } }) // A manually configured recovery (bootstrap.enabled without // newServerName) renders no annotation. Re-rendered after StartedAt but // before this restore patches the values, it holds the manual recovery // source, not this restore's backup. t.Run("unmarked cluster from values this restore has not patched is purged even when created after start", func(t *testing.T) { backup := mkBackupArtifact(t) fresh := metav1.NewTime(startedAt.Add(time.Minute)) app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = true app.Spec.Bootstrap.OldName = "manual-source" c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, app, mkRecoveryCluster(fresh), mkClusterPVC()) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}) if !apierrors.IsNotFound(err) { t.Fatalf("expected the manual recovery Cluster to be purged (NotFound), got err=%v", err) } }) t.Run("recovery cluster marked by another restore is purged even when created after start", func(t *testing.T) { backup := mkBackupArtifact(t) fresh := metav1.NewTime(startedAt.Add(time.Minute)) other := mkRecoveryCluster(fresh) other.Annotations = map[string]string{cnpgRestoredServerNameAnnotation: clusterName + "-restore-0123456789abcdef"} c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, newPostgresApp(appName, ns), other, mkClusterPVC()) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}) if !apierrors.IsNotFound(err) { t.Fatalf("expected another restore's Cluster to be purged (NotFound), got err=%v", err) } }) // The mark the purge decision expects has to be the one the restore patch // writes into bootstrap.newServerName, which the chart renders onto the // Cluster. The Cluster here predates StartedAt, so only a matching mark // keeps it: the creation-time fallback would purge it. t.Run("recovery cluster carrying the mark this restore's patch wrote is not re-purged", func(t *testing.T) { backup := mkBackupArtifact(t) stale := metav1.NewTime(startedAt.Add(-time.Hour)) c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, newPostgresApp(appName, ns), mkRecoveryCluster(stale), mkClusterPVC()) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("first reconcileCNPGRestore: %v", err) } if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}); !apierrors.IsNotFound(err) { t.Fatalf("first pass should purge the stale Cluster, got err=%v", err) } app := &postgresapp.Postgres{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, app); err != nil { t.Fatalf("get patched Postgres app: %v", err) } mark := app.Spec.Bootstrap.NewServerName if mark == "" { t.Fatalf("the restore patch left bootstrap.newServerName empty") } // The chart re-renders the Cluster from the patched values, and the // TargetPurged status write is lost. rendered := mkRecoveryCluster(stale) rendered.Annotations = map[string]string{cnpgRestoredServerNameAnnotation: mark} if err := c.Create(ctx, rendered); err != nil { t.Fatalf("create re-rendered Cluster: %v", err) } if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("reload RestoreJob: %v", err) } apimeta.RemoveStatusCondition(&rj.Status.Conditions, restoreCondTargetPurged) if err := c.Status().Update(ctx, rj); err != nil { t.Fatalf("drop TargetPurged: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("second reconcileCNPGRestore: %v", err) } got := &cnpgtypes.Cluster{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, got); err != nil { t.Fatalf("expected the Cluster this restore rendered to survive, got err=%v", err) } if !got.DeletionTimestamp.IsZero() { t.Fatalf("the Cluster this restore rendered must not be marked for deletion") } }) } // TestReconcileCNPGRestore_RefusesCrossFlavorRestore: a physical backup only // restores onto the image family it was taken from. The postgresql and // postgis images ship different glibc versions, and postgis objects need the // postgis libraries, so a cross-flavor RestoreJob must fail before the purge // deletes the target's Cluster and PVCs. A snapshot without a flavor predates // flavors and therefore comes from a postgresql cluster. func TestReconcileCNPGRestore_RefusesCrossFlavorRestore(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() startedAt := metav1.NewTime(time.Now()) strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{ Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }, }, } cnpgBackup := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003"}, } cases := []struct { name string sourceFlavor string targetFlavor string wantFailed bool wantInMessage string }{ {name: "postgis backup into a postgresql app", sourceFlavor: "postgis", targetFlavor: "", wantFailed: true, wantInMessage: "taken from a postgis cluster"}, {name: "pre-flavor backup into a postgis app", sourceFlavor: "", targetFlavor: "postgis", wantFailed: true, wantInMessage: "taken from a postgresql cluster"}, {name: "postgis backup into a postgis app", sourceFlavor: "postgis", targetFlavor: "postgis", wantFailed: false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, tc.sourceFlavor) if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } target := newPostgresApp(appName, ns) target.Spec.Flavor = tc.targetFlavor sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, } liveCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(-time.Hour))}, } pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName + "-1", Labels: map[string]string{cnpgClusterLabel: clusterName}}, } c := newCNPGStrategyTestClient(t, backup, restoreJob, strategy, cnpgBackup, target, liveCluster, pvc) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } got := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil { t.Fatalf("get RestoreJob: %v", err) } clusterErr := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}) app, err := r.getPostgresApp(ctx, ns, appName) if err != nil { t.Fatalf("get target app: %v", err) } if !tc.wantFailed { if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("same-flavor restore must proceed, got Failed: %s", got.Status.Message) } if !apierrors.IsNotFound(clusterErr) { t.Fatalf("same-flavor restore must purge the target Cluster, got err=%v", clusterErr) } return } if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("cross-flavor restore must fail, got phase %q", got.Status.Phase) } if !strings.Contains(got.Status.Message, tc.wantInMessage) || !strings.Contains(got.Status.Message, "pg_dump") { t.Errorf("failure message must name the source flavor and point to a logical dump, got %q", got.Status.Message) } if clusterErr != nil { t.Fatalf("cross-flavor restore must leave the target Cluster in place, got err=%v", clusterErr) } pvcs := &corev1.PersistentVolumeClaimList{} if err := c.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{cnpgClusterLabel: clusterName}); err != nil { t.Fatalf("list PVCs: %v", err) } if len(pvcs.Items) != 1 { t.Fatalf("cross-flavor restore must leave the target PVCs in place, have %d", len(pvcs.Items)) } if app.Spec.Bootstrap.Enabled { t.Fatalf("cross-flavor restore must not patch the target app for recovery") } }) } } // TestReconcileCNPGRestore_HealthyClusterSucceeds drives the reconcile to its // success branch: with the target already purged (TargetPurged=True) and the // recovery Cluster reporting healthy, the RestoreJob must reach phase // Succeeded with Ready=True AND the new RecoveryConverged=True condition. func TestReconcileCNPGRestore_HealthyClusterSucceeds(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() startedAt := metav1.NewTime(time.Now()) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } // Healthy recovery Cluster created after StartedAt (this restore's own). healthyCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, Status: cnpgtypes.ClusterStatus{Phase: cnpgClusterHealthyPhase}, } sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, // Purge already done for this restore, so reconcile skips straight // to the wait-for-healthy / success path. A prior reconcile hit a // transient disable error and recorded BootstrapDisablePending=True // within the grace window; the disable now succeeds, so the terminal // success write must clear that pending condition rather than leave it // standing True next to a Succeeded RestoreJob. Conditions: []metav1.Condition{ { Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged", }, { Type: restoreCondBootstrapDisablePending, Status: metav1.ConditionTrue, Reason: "Retrying", LastTransitionTime: startedAt, Message: "clearing spec.bootstrap.enabled is retrying after a transient error", }, }, }, } c := newCNPGStrategyTestClient(t, backup, restoreJob, strategy, newPostgresApp(appName, ns), healthyCluster) rec := record.NewFakeRecorder(10) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: rec} // A healthy recovery now converges in two reconciles: the first latches // RecoveryConverged durably and requeues, the second disables bootstrap and // marks Succeeded (see reconcileCNPGRestore's terminal block). got := reconcileCNPGRestoreToTerminal(t, ctx, r, c, backup, ns, "rj") if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { t.Fatalf("expected phase Succeeded, got %q (msg=%q)", got.Status.Phase, got.Status.Message) } if c := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); c == nil || c.Status != metav1.ConditionTrue { t.Fatalf("expected Ready=True, got %+v", c) } if c := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondRecoveryConverged); c == nil || c.Status != metav1.ConditionTrue { t.Fatalf("expected RecoveryConverged=True, got %+v", c) } // The terminal write must also emit the pending-convergence Event, so the // credential handoff is discoverable in `kubectl get events`, not only in // .status. Dropping the r.Recorder.Eventf call leaves this channel empty. select { case ev := <-rec.Events: if !strings.Contains(ev, "CredentialsConvergencePending") { t.Fatalf("expected a CredentialsConvergencePending event, got %q", ev) } default: t.Fatalf("expected a CredentialsConvergencePending event, got none") } // The pending credential handoff is carried as the REASON on Ready=True (not a // standalone never-True condition): a durable, readable marker that does not // look like a standing failure. Changing the reason back to a bare // "RestoreCompleted" reddens this. if c := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); c == nil || c.Reason != "RestoreCompletedCredentialsPending" { t.Fatalf("expected Ready reason RestoreCompletedCredentialsPending, got %+v", c) } // The seeded BootstrapDisablePending=True must be closed out on success, not // left standing next to a Succeeded RestoreJob. Dropping the terminal // clear-block reddens this. if c := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondBootstrapDisablePending); c == nil || c.Status != metav1.ConditionFalse || c.Reason != "Cleared" { t.Fatalf("expected BootstrapDisablePending cleared to False/Cleared, got %+v", c) } } // TestReconcileCNPGRestore_HealthyDisablesBootstrap is the regression guard for // the restore-credentials fix: patchPostgresAppForRestore leaves the target app // with bootstrap.enabled=true, and while it stays true the chart skips the // init-job that reconciles the generated -credentials Secret onto the // recovered roles. The recovered roles carry the source's password hashes, the // freshly generated Secret does not match them, so without this flip every // application-user login against the restored target fails. On convergence the // controller must clear bootstrap.enabled so the next HelmRelease upgrade runs // that init-job. func TestReconcileCNPGRestore_HealthyDisablesBootstrap(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() startedAt := metav1.NewTime(time.Now().Add(-time.Hour)) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } healthyCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, Status: cnpgtypes.ClusterStatus{Phase: cnpgClusterHealthyPhase}, } // The target app is mid-restore: patchPostgresAppForRestore already set // bootstrap.enabled=true (oldName is required alongside it by the schema). app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = true app.Spec.Bootstrap.OldName = appName sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, Conditions: []metav1.Condition{{ Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged", }}, }, } c := newCNPGStrategyTestClient(t, backup, restoreJob, strategy, app, healthyCluster) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} got := reconcileCNPGRestoreToTerminal(t, ctx, r, c, backup, ns, "rj") if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { t.Fatalf("expected phase Succeeded, got %q (msg=%q)", got.Status.Phase, got.Status.Message) } gotApp := &postgresapp.Postgres{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, gotApp); err != nil { t.Fatalf("get target app after reconcile: %v", err) } if gotApp.Spec.Bootstrap.Enabled { t.Fatalf("expected bootstrap.enabled cleared after recovery converged, still true") } // oldName must survive: the app schema requires it whenever the bootstrap // block is present, and clearing it would fail admission on the next apply. if gotApp.Spec.Bootstrap.OldName != appName { t.Fatalf("expected bootstrap.oldName preserved, got %q", gotApp.Spec.Bootstrap.OldName) } } // TestReconcileCNPGRestore_BootstrapDisableFailsPastDeadline is the regression // guard for the still-open risk on the disable-bootstrap terminal step: recovery // has converged (RecoveryConverged=True latched) but clearing bootstrap.enabled // keeps failing - the app was deleted/renamed mid-restore, or a GitOps source // re-asserts it. Without a deadline bound the terminal block requeues forever and // the RestoreJob sits Running with no Failed/Ready=False. Past the restore // deadline it must terminate as Failed with a BootstrapDisableFailed reason. func TestReconcileCNPGRestore_BootstrapDisableFailsPastDeadline(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() // Well past the 30m default deadline. startedAt := metav1.NewTime(time.Now().Add(-72 * time.Hour)) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } healthyCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, Status: cnpgtypes.ClusterStatus{Phase: cnpgClusterHealthyPhase}, } app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = true app.Spec.Bootstrap.OldName = appName sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, Conditions: []metav1.Condition{ {Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged"}, {Type: restoreCondRecoveryConverged, Status: metav1.ConditionTrue, Reason: "RecoveryConverged", LastTransitionTime: startedAt, Message: "converged"}, }, }, } // A fake client that fails Patch on the Postgres app, so // disablePostgresAppBootstrap can never clear bootstrap.enabled. s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = cnpgtypes.AddToScheme(s) _ = postgresapp.AddToScheme(s) c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(backup, restoreJob, strategy, app, healthyCluster). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). WithInterceptorFuncs(interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*postgresapp.Postgres); ok { return fmt.Errorf("injected bootstrap-disable failure") } return cl.Patch(ctx, obj, patch, opts...) }, }). Build() r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } got := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil { t.Fatalf("get RestoreJob after reconcile: %v", err) } if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("expected phase Failed when bootstrap-disable keeps failing past the deadline, got %q", got.Status.Phase) } if cond := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); cond == nil || cond.Reason != "BootstrapDisableFailed" { t.Fatalf("expected Ready=False reason BootstrapDisableFailed, got %+v", cond) } } // TestReconcileCNPGRestore_BootstrapDisableTransientErrorRequeues guards the // grace window on the disable step: the health check wins BEFORE the restore // deadline on purpose, so recovery may legitimately have spent the whole // deadline by the time bootstrap is cleared. The disable step must therefore be // bounded from RecoveryConverged's LastTransitionTime, not from StartedAt - or a // single transient error on the first disable attempt (StartedAt already past // the deadline) would terminate the restore Failed, which on a resubmit lets the // purge-guard delete the healthy restored Cluster. A transient failure right // after convergence must requeue, not fail. func TestReconcileCNPGRestore_BootstrapDisableTransientErrorRequeues(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() // StartedAt is well past the 30m restore deadline: a StartedAt-based bound // would fail immediately; convergence just happened, so the disable window is // still open and a transient error must requeue. startedAt := metav1.NewTime(time.Now().Add(-72 * time.Hour)) convergedAt := metav1.Now() snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } healthyCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, Status: cnpgtypes.ClusterStatus{Phase: cnpgClusterHealthyPhase}, } app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = true app.Spec.Bootstrap.OldName = appName sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, Conditions: []metav1.Condition{ {Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged"}, {Type: restoreCondRecoveryConverged, Status: metav1.ConditionTrue, Reason: "RecoveryConverged", LastTransitionTime: convergedAt, Message: "converged"}, }, }, } s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = cnpgtypes.AddToScheme(s) _ = postgresapp.AddToScheme(s) c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(backup, restoreJob, strategy, app, healthyCluster). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). WithInterceptorFuncs(interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*postgresapp.Postgres); ok { return fmt.Errorf("injected transient bootstrap-disable failure") } return cl.Patch(ctx, obj, patch, opts...) }, }). Build() rec := record.NewFakeRecorder(10) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: rec} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } res, err := r.reconcileCNPGRestore(ctx, rj, backup) if err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } if res.RequeueAfter <= 0 { t.Fatalf("expected a requeue after a transient disable failure, got %+v", res) } got := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil { t.Fatalf("get RestoreJob after reconcile: %v", err) } if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("a transient disable failure within the post-convergence window must requeue, not Fail; got phase %q", got.Status.Phase) } // The window must not be silent: the requeue records WHY on .status (a durable // condition) and announces it once via an Event. Dropping either leaves the // RestoreJob non-terminal for up to the whole grace window with no explanation. if c := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondBootstrapDisablePending); c == nil || c.Status != metav1.ConditionTrue { t.Fatalf("expected BootstrapDisablePending=True during the window, got %+v", c) } select { case ev := <-rec.Events: if !strings.Contains(ev, "BootstrapDisablePending") { t.Fatalf("expected a BootstrapDisablePending event, got %q", ev) } default: t.Fatalf("expected a BootstrapDisablePending event, got none") } } // TestReconcileCNPGRestore_BootstrapDisableShortTimeoutHonorsGraceFloor is the // regression guard for the grace-window floor. A tenant sets // restoreTimeoutSeconds:60 to fail fast on a stuck recovery; recovery actually // converged 90s ago and the disable step is now hitting a transient error. // Without the floor the disable grace collapses to 60s (the recovery timeout), // so 90s-since-convergence terminates the restore Failed on a blip - and a // resubmit's purge-guard would then delete the healthy restored Cluster. With // the floor (cnpgBootstrapDisableGrace) the window stays open and the // transient error requeues. Reverting effectiveBootstrapDisableGrace back to // effectiveRestoreDeadline turns this red. func TestReconcileCNPGRestore_BootstrapDisableShortTimeoutHonorsGraceFloor(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() startedAt := metav1.NewTime(time.Now().Add(-72 * time.Hour)) // Converged 90s ago: past the tenant's 60s restoreTimeoutSeconds but well // within the 5m post-convergence floor. convergedAt := metav1.NewTime(time.Now().Add(-90 * time.Second)) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } healthyCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, Status: cnpgtypes.ClusterStatus{Phase: cnpgClusterHealthyPhase}, } app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = true app.Spec.Bootstrap.OldName = appName sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: "bk"}, Options: &runtime.RawExtension{Raw: []byte(`{"restoreTimeoutSeconds":60}`)}, }, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, Conditions: []metav1.Condition{ {Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged"}, {Type: restoreCondRecoveryConverged, Status: metav1.ConditionTrue, Reason: "RecoveryConverged", LastTransitionTime: convergedAt, Message: "converged"}, }, }, } s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = cnpgtypes.AddToScheme(s) _ = postgresapp.AddToScheme(s) c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(backup, restoreJob, strategy, app, healthyCluster). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). WithInterceptorFuncs(interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*postgresapp.Postgres); ok { return fmt.Errorf("injected transient bootstrap-disable failure") } return cl.Patch(ctx, obj, patch, opts...) }, }). Build() r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } res, err := r.reconcileCNPGRestore(ctx, rj, backup) if err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } if res.RequeueAfter <= 0 { t.Fatalf("expected a requeue within the floored grace window, got %+v", res) } got := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil { t.Fatalf("get RestoreJob after reconcile: %v", err) } if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("a short restoreTimeoutSeconds must not shrink the disable grace below the floor; a blip 90s after convergence must requeue, got phase %q", got.Status.Phase) } } // TestReconcileCNPGRestore_HealthyPastDeadlineSucceeds is the regression guard // for the health-before-deadline ordering: a recovery that has reached a // healthy Cluster must be reported Succeeded even when the reconcile observing // it lands AFTER the restore deadline has elapsed (e.g. a controller restart // spanning the window). StartedAt is 72h in the past (well past the 30m default // deadline) and a recoveryTime is set, so if the deadline branch ran first it // would classify the restore Failed/RecoveryTargetUnreachable - a wrong verdict // that can escalate to data loss on resubmit. The injected readPodLog fails the // test if the classification path is entered at all, proving health short-circuits. func TestReconcileCNPGRestore_HealthyPastDeadlineSucceeds(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() // Far past the 30m default deadline: this is the reconcile-gap case. startedAt := metav1.NewTime(time.Now().Add(-72 * time.Hour)) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } healthyCluster := &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, Status: cnpgtypes.ClusterStatus{Phase: cnpgClusterHealthyPhase}, } sa := startedAt restoreJob := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: "bk"}, // recoveryTime set so the (wrongly-ordered) deadline branch would classify. Options: &runtime.RawExtension{Raw: []byte(`{"recoveryTime":"2026-01-01T00:00:00Z"}`)}, }, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, Conditions: []metav1.Condition{{ Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged", }}, }, } c := newCNPGStrategyTestClient(t, backup, restoreJob, strategy, newPostgresApp(appName, ns), healthyCluster) r := &RestoreJobReconciler{ Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10), // If the deadline/classification branch runs before the health check, // it reaches the log reader - which must never happen for a healthy cluster. readPodLog: func(context.Context, string, string, string) (string, error) { t.Fatalf("classification (log read) ran for an already-healthy Cluster: health check must precede the deadline branch") return "", nil }, } // Two reconciles to terminal (latch, then disable-bootstrap + Succeeded); // the injected readPodLog still fails the test if the deadline/classification // branch runs on either pass, proving health short-circuits regardless. got := reconcileCNPGRestoreToTerminal(t, ctx, r, c, backup, ns, "rj") if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { t.Fatalf("expected phase Succeeded (healthy cluster past deadline), got %q (msg=%q)", got.Status.Phase, got.Status.Message) } if cond := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionTrue { t.Fatalf("expected Ready=True, got %+v", cond) } if cond := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondRecoveryConverged); cond == nil || cond.Status != metav1.ConditionTrue { t.Fatalf("expected RecoveryConverged=True, got %+v", cond) } } // TestReconcileCNPGRestore_DeadlineClassifiesRecoveryTargetUnreachable drives the // full reconcileCNPGRestore deadline branch (the classification call site), which // the pure-helper tests (TestRecoveryTargetUnreachable_*) do not exercise: // swapping the RecoveryTargetUnreachable reason back to the generic RestoreFailed // at that call site leaves them green. Each sub-case backdates StartedAt past the // restore deadline with a recoveryTime set (so classification runs) and asserts // the three outcomes the branch distinguishes: FATAL found, FATAL absent, and a // Forbidden log read (missing pods/log RBAC). func TestReconcileCNPGRestore_DeadlineClassifiesRecoveryTargetUnreachable(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() // StartedAt far past cnpgDefaultRestoreDeadline (30m) so the first reconcile // enters the deadline-classification branch. startedAt := metav1.NewTime(time.Now().Add(-72 * time.Hour)) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } mkBackup := func() *backupsv1alpha1.Backup { return &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", }, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }}, } // Recovery Cluster created after StartedAt (this restore's own -> not // re-purged), so the reconcile flows past the purge block to the deadline // check. Phase left empty (still wedged): the deadline check precedes the // healthy check, so it fires regardless. mkCluster := func() *cnpgtypes.Cluster { return &cnpgtypes.Cluster{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(time.Minute))}, Spec: cnpgtypes.ClusterSpec{Bootstrap: &cnpgtypes.BootstrapConfiguration{Recovery: &cnpgtypes.RecoverySource{Source: appName}}}, } } // A bootstrap-recovery pod for the cluster; recoveryTargetUnreachable lists // these and reads each through the readPodLog seam. mkPod := func() *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: clusterName + "-full-recovery-abcde", Labels: map[string]string{cnpgClusterLabel: clusterName}, CreationTimestamp: metav1.NewTime(startedAt.Add(2 * time.Minute)), }, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: cnpgRecoveryContainerName}}}, } } mkRestoreJob := func() *backupsv1alpha1.RestoreJob { sa := startedAt return &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: "bk"}, // recoveryTime present -> the reconcile runs the classification. Options: &runtime.RawExtension{Raw: []byte(`{"recoveryTime":"2026-01-01T00:00:00Z"}`)}, }, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning, // Purge already done -> reconcile reaches the deadline check. Conditions: []metav1.Condition{{ Type: restoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged", LastTransitionTime: startedAt, Message: "purged", }}, }, } } fatalLog := "LOG: redo done\nFATAL: " + cnpgRecoveryTargetUnreachableLog newReconciler := func(c client.Client, readLog func(context.Context, string, string, string) (string, error)) *RestoreJobReconciler { return &RestoreJobReconciler{ Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10), readPodLog: readLog, } } reconcileToFailed := func(t *testing.T, r *RestoreJobReconciler, c client.Client, backup *backupsv1alpha1.Backup) *backupsv1alpha1.RestoreJob { t.Helper() rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } res, err := r.reconcileCNPGRestore(ctx, rj, backup) if err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } if res.RequeueAfter != 0 { t.Fatalf("terminal deadline failure must not requeue, got %+v", res) } got := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil { t.Fatalf("get RestoreJob after reconcile: %v", err) } if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("expected Phase=Failed after deadline, got %q (msg=%q)", got.Status.Phase, got.Status.Message) } return got } t.Run("recovery-target-unreachable FATAL -> RecoveryTargetUnreachable reason", func(t *testing.T) { backup := mkBackup() c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, newPostgresApp(appName, ns), mkCluster(), mkPod()) r := newReconciler(c, func(context.Context, string, string, string) (string, error) { return fatalLog, nil }) got := reconcileToFailed(t, r, c, backup) // The mutation the review flagged: the call site must set reason // RecoveryTargetUnreachable, not the generic RestoreFailed. conv := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondRecoveryConverged) if conv == nil || conv.Status != metav1.ConditionFalse || conv.Reason != "RecoveryTargetUnreachable" { t.Fatalf("expected RecoveryConverged=False reason=RecoveryTargetUnreachable, got %+v", conv) } if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready == nil || ready.Reason != "RecoveryTargetUnreachable" { t.Fatalf("expected Ready reason=RecoveryTargetUnreachable, got %+v", ready) } if !strings.Contains(got.Status.Message, "past the latest WAL archived") { t.Errorf("message should explain the unreachable target, got %q", got.Status.Message) } }) t.Run("no FATAL -> generic deadline failure, not RecoveryTargetUnreachable", func(t *testing.T) { backup := mkBackup() c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, newPostgresApp(appName, ns), mkCluster(), mkPod()) r := newReconciler(c, func(context.Context, string, string, string) (string, error) { return "LOG: consistent recovery state reached", nil }) got := reconcileToFailed(t, r, c, backup) if conv := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondRecoveryConverged); conv != nil { t.Fatalf("did not expect a RecoveryConverged condition on a generic timeout, got %+v", conv) } if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready != nil && ready.Reason == "RecoveryTargetUnreachable" { t.Fatalf("generic timeout must NOT be classified RecoveryTargetUnreachable, got %+v", ready) } if !strings.Contains(got.Status.Message, "deadline") { t.Errorf("message should mention the deadline, got %q", got.Status.Message) } if strings.Contains(got.Status.Message, "pods/log RBAC") { t.Errorf("a readable log must not report a Forbidden RBAC note, got %q", got.Status.Message) } }) t.Run("Forbidden log read -> generic failure with missing-RBAC note", func(t *testing.T) { backup := mkBackup() c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, newPostgresApp(appName, ns), mkCluster(), mkPod()) r := newReconciler(c, func(context.Context, string, string, string) (string, error) { return "", apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "rec", fmt.Errorf("pods/log grant missing")) }) got := reconcileToFailed(t, r, c, backup) if conv := apimeta.FindStatusCondition(got.Status.Conditions, restoreCondRecoveryConverged); conv != nil && conv.Reason == "RecoveryTargetUnreachable" { t.Fatalf("a Forbidden log read must not yield a RecoveryTargetUnreachable classification, got %+v", conv) } if !strings.Contains(got.Status.Message, "pods/log RBAC") { t.Errorf("Forbidden classification should surface the missing pods/log RBAC grant, got %q", got.Status.Message) } }) } // newCNPGStrategyTestClient returns a fake client.Client wired up with // the schemes the CNPG-strategy reconciler needs. func newCNPGStrategyTestClient(t *testing.T, objs ...client.Object) client.Client { t.Helper() s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = cnpgtypes.AddToScheme(s) _ = postgresapp.AddToScheme(s) return clientfake.NewClientBuilder(). WithScheme(s). WithObjects(objs...). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). Build() } // reconcileCNPGRestoreToTerminal drives reconcileCNPGRestore until the // RestoreJob reaches a terminal phase (Succeeded/Failed) or a small iteration // bound is hit. The success path is multi-step - it latches RecoveryConverged // on one reconcile and disables bootstrap + marks Succeeded on the next - so a // single reconcile does not reach the verdict. Re-fetching between passes // mirrors how the controller-runtime requeue would re-enter Reconcile. func reconcileCNPGRestoreToTerminal(t *testing.T, ctx context.Context, r *RestoreJobReconciler, c client.Client, backup *backupsv1alpha1.Backup, ns, name string) *backupsv1alpha1.RestoreJob { t.Helper() got := &backupsv1alpha1.RestoreJob{} for i := 0; i < 8; i++ { rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, rj); err != nil { t.Fatalf("get RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, got); err != nil { t.Fatalf("get RestoreJob after reconcile: %v", err) } if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { return got } } t.Fatalf("RestoreJob did not reach a terminal phase; last phase %q", got.Status.Phase) return got } // TestDisablePostgresAppBootstrap_Idempotent pins the early-return in // disablePostgresAppBootstrap. When the app already has bootstrap.enabled=false // the method must return without issuing a Patch: the already-disabled subtest // fails any Patch on the Postgres app, so removing the `if !Enabled { return nil }` // guard turns the intended no-op into a failed Patch and reddens it. When the flag // is true the method must patch it to false. func TestDisablePostgresAppBootstrap_Idempotent(t *testing.T) { const ( ns = "tenant" appName = "app" ) ctx := context.Background() s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = postgresapp.AddToScheme(s) t.Run("already disabled is a no-op that issues no Patch", func(t *testing.T) { app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = false c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(app). WithInterceptorFuncs(interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*postgresapp.Postgres); ok { return fmt.Errorf("Patch must not be issued when bootstrap is already disabled") } return cl.Patch(ctx, obj, patch, opts...) }, }). Build() r := &RestoreJobReconciler{Client: c} if err := r.disablePostgresAppBootstrap(ctx, ns, appName); err != nil { t.Fatalf("expected a no-op returning nil, got %v", err) } }) t.Run("enabled is patched to false", func(t *testing.T) { app := newPostgresApp(appName, ns) app.Spec.Bootstrap.Enabled = true c := clientfake.NewClientBuilder().WithScheme(s).WithObjects(app).Build() r := &RestoreJobReconciler{Client: c} if err := r.disablePostgresAppBootstrap(ctx, ns, appName); err != nil { t.Fatalf("disable: %v", err) } got := &postgresapp.Postgres{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, got); err != nil { t.Fatalf("get app after disable: %v", err) } if got.Spec.Bootstrap.Enabled { t.Fatalf("expected bootstrap.enabled=false after disable, still true") } }) } // TestCNPGRestoreBackupID pins when a restore names the base backup it starts // from. Without a backupID the barman-cloud plugin picks the newest backup of // the stream (no recoveryTime) or the newest one ending by recoveryTime, on // any timeline, which is not necessarily the backup the RestoreJob names. func TestCNPGRestoreBackupID(t *testing.T) { stopped := metav1.NewTime(time.Date(2026, 9, 21, 6, 27, 23, 0, time.UTC)) cases := []struct { name string backupID string stoppedAt *metav1.Time recoveryTime string want string }{ {"no recoveryTime pins the requested backup", "20260921T062713", &stopped, "", "20260921T062713"}, {"no recoveryTime pins it even without its end", "20260921T062713", nil, "", "20260921T062713"}, {"recoveryTime a second past the end pins it", "20260921T062713", &stopped, "2026-09-21T06:27:24Z", "20260921T062713"}, {"recoveryTime well after the end pins it", "20260921T062713", &stopped, "2026-09-22T10:00:00+02:00", "20260921T062713"}, {"recoveryTime within the truncated second leaves the choice to the plugin", "20260921T062713", &stopped, "2026-09-21T06:27:23.5Z", ""}, {"recoveryTime before the end leaves the choice to the plugin", "20260921T062713", &stopped, "2026-09-21T05:00:00Z", ""}, {"unknown end with a recoveryTime leaves the choice to the plugin", "20260921T062713", nil, "2026-09-22T10:00:00Z", ""}, {"unparseable recoveryTime leaves the choice to the plugin", "20260921T062713", &stopped, "2026-09-22 10:00:00", ""}, {"unknown backupID pins nothing", "", &stopped, "", ""}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { source := &cnpgtypes.Backup{Status: cnpgtypes.BackupStatus{BackupID: tc.backupID, StoppedAt: tc.stoppedAt}} if got := cnpgRestoreBackupID(source, tc.recoveryTime); got != tc.want { t.Errorf("cnpgRestoreBackupID(%q, %v, %q) = %q, want %q", tc.backupID, tc.stoppedAt, tc.recoveryTime, got, tc.want) } }) } t.Run("a cnpg.io/Backup that is gone pins nothing", func(t *testing.T) { if got := cnpgRestoreBackupID(nil, ""); got != "" { t.Errorf("cnpgRestoreBackupID(nil, \"\") = %q, want none", got) } }) } // TestBuildPostgresAppRestorePatch_ReplacesBackupID guards against a // re-restore into the same target starting from the backup a previous // restore pinned. func TestBuildPostgresAppRestorePatch_ReplacesBackupID(t *testing.T) { app := newPostgresApp("pg", "tenant") app.Spec.Bootstrap.BackupID = "20260101T000000" if got := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "20260921T062713", nil, nil, nil, nil).Spec.Bootstrap.BackupID; got != "20260921T062713" { t.Errorf("bootstrap.backupID: got %q want %q", got, "20260921T062713") } if got := buildPostgresAppRestorePatch(app, "src", "pg-new", "s3://b/", "", "", "", nil, nil, nil, nil).Spec.Bootstrap.BackupID; got != "" { t.Errorf("bootstrap.backupID from a previous restore must be cleared; got %q", got) } } // TestReconcileCNPGRestore_PinsTheRequestedBackup drives a restore through the // purge step and reads back what the target Postgres app is patched with: the // barman backupId of the backup the RestoreJob names, read from its // cnpg.io/Backup. Retention deletes that cnpg.io/Backup together with the base // backup, and pinning an id the catalog has lost fails the recovery after the // purge, so a cnpg.io/Backup that is gone leaves the choice to the plugin. func TestReconcileCNPGRestore_PinsTheRequestedBackup(t *testing.T) { const ( ns = "tenant" appName = "app" clusterName = "postgres-app" cnpgBkName = "cnpgbk" backupID = "20260921T062713" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() startedAt := metav1.NewTime(time.Now()) stoppedAt := metav1.NewTime(time.Date(2026, 9, 21, 6, 27, 23, 0, time.UTC)) mkBackupArtifact := func(t *testing.T) *backupsv1alpha1.Backup { t.Helper() snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } driverMD := map[string]string{ cnpgServerNameKey: appName, cnpgDestinationPathKey: "s3://bucket/" + appName + "/", cnpgBackupNameKey: cnpgBkName, } return &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: driverMD, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } } strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{ Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }, }, } mkCNPGBackup := func(id string) *cnpgtypes.Backup { return &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, Status: cnpgtypes.BackupStatus{ Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003", BackupID: id, StoppedAt: &stoppedAt, }, } } mkRestoreJob := func(recoveryTime string) *backupsv1alpha1.RestoreJob { sa := startedAt rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, } if recoveryTime != "" { rj.Spec.Options = &runtime.RawExtension{Raw: []byte(`{"recoveryTime":"` + recoveryTime + `"}`)} } return rj } restoredBootstrap := func(t *testing.T, cnpgBackup *cnpgtypes.Backup, recoveryTime string) (postgresapp.Bootstrap, *record.FakeRecorder) { t.Helper() backup := mkBackupArtifact(t) objs := []client.Object{backup, mkRestoreJob(recoveryTime), strategy, newPostgresApp(appName, ns)} if cnpgBackup != nil { objs = append(objs, cnpgBackup) } c := newCNPGStrategyTestClient(t, objs...) recorder := record.NewFakeRecorder(10) r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: recorder} rj := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } app := &postgresapp.Postgres{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, app); err != nil { t.Fatalf("get patched Postgres app: %v", err) } if !app.Spec.Bootstrap.Enabled { t.Fatalf("the reconcile did not reach the restore patch") } return app.Spec.Bootstrap, recorder } t.Run("without recoveryTime the requested backup is pinned", func(t *testing.T) { if b, _ := restoredBootstrap(t, mkCNPGBackup(backupID), ""); b.BackupID != backupID { t.Errorf("bootstrap.backupID: got %q want %q", b.BackupID, backupID) } }) t.Run("a recoveryTime after the backup ended pins it", func(t *testing.T) { b, _ := restoredBootstrap(t, mkCNPGBackup(backupID), "2026-09-22T10:00:00Z") if b.BackupID != backupID || b.RecoveryTime != "2026-09-22T10:00:00Z" { t.Errorf("bootstrap: got backupID %q recoveryTime %q, want %q and the requested instant", b.BackupID, b.RecoveryTime, backupID) } }) t.Run("a recoveryTime before the backup ended leaves the choice to the plugin", func(t *testing.T) { if b, _ := restoredBootstrap(t, mkCNPGBackup(backupID), "2026-09-21T05:00:00Z"); b.BackupID != "" { t.Errorf("bootstrap.backupID: got %q, want none: the backup cannot serve an instant before its end", b.BackupID) } }) for _, recoveryTime := range []string{"", "2026-09-22T10:00:00Z"} { t.Run("a cnpg.io/Backup removed by retention leaves the choice to the plugin, recoveryTime="+recoveryTime, func(t *testing.T) { b, recorder := restoredBootstrap(t, nil, recoveryTime) if b.BackupID != "" { t.Errorf("bootstrap.backupID: got %q, want none: the catalog no longer holds that base backup", b.BackupID) } select { case ev := <-recorder.Events: if !strings.Contains(ev, "Warning BaseBackupGone") { t.Errorf("event: got %q, want a BaseBackupGone warning", ev) } default: t.Errorf("no event: a restore that cannot start from the backup it names must say so") } }) } } func TestCreateCNPGBackupArtifact_RecordsMajorVersion(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group for _, tc := range []struct { major int want string }{{major: 18, want: "18"}, {major: 0, want: ""}} { t.Run(fmt.Sprintf("majorVersion=%d", tc.major), func(t *testing.T) { r := &BackupJobReconciler{Client: newCNPGStrategyTestClient(t)} j := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, }, } resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"}, } cnpgBk := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{MajorVersion: tc.major}, } rendered := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}, } got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } if v, ok := got.Spec.DriverMetadata[cnpgMajorVersionKey]; v != tc.want || ok != (tc.want != "") { t.Errorf("driverMetadata[%s]: got %q (present=%v), want %q", cnpgMajorVersionKey, v, ok, tc.want) } }) } } // TestReconcileCNPGRestore_RefusesCrossMajorRestore drives the restore up to // the purge with the backup and the target on different PostgreSQL majors. // The purge used to run regardless, and the recovery could then never start: // the target was left empty. func TestReconcileCNPGRestore_RefusesCrossMajorRestore(t *testing.T) { const ( ns = "tenant" sourceApp = "app" copyApp = "copy" cnpgBkName = "cnpgbk" ) apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group ctx := context.Background() startedAt := metav1.NewTime(time.Now()) strategy := &strategyv1alpha1.CNPG{ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, Spec: strategyv1alpha1.CNPGSpec{ Template: strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, }, }, } for _, tc := range []struct { name string // recorded is the major the artifact's driverMetadata carries, "" for // an artifact taken before the driver recorded it. recorded string // cnpgMajor is the cnpg.io/Backup's status.majorVersion; -1 for a // cnpg.io/Backup that retention has removed. cnpgMajor int targetVersion string toCopy bool wantRefused bool }{ {name: "v18 backup into a v16 copy", recorded: "18", cnpgMajor: 18, targetVersion: "v16", toCopy: true, wantRefused: true}, {name: "in place into an app upgraded since its v16 backup", recorded: "16", cnpgMajor: 16, targetVersion: "v18", wantRefused: true}, {name: "artifact without the record falls back to the cnpg.io/Backup", cnpgMajor: 18, targetVersion: "v16", toCopy: true, wantRefused: true}, {name: "same major restores", recorded: "16", cnpgMajor: 16, targetVersion: "v16", toCopy: true}, {name: "unknown backup major restores", cnpgMajor: -1, targetVersion: "v16", toCopy: true}, {name: "cnpg.io/Backup without majorVersion restores", cnpgMajor: 0, targetVersion: "v16"}, {name: "unknown target version restores", recorded: "18", cnpgMajor: 18, targetVersion: ""}, } { t.Run(tc.name, func(t *testing.T) { targetApp := sourceApp if tc.toCopy { targetApp = copyApp } targetCluster := cnpgClusterNameForApp(targetApp) snap, err := marshalCNPGBackupSnapshot(newPostgresApp(sourceApp, ns), nil, "") if err != nil { t.Fatalf("marshal snapshot: %v", err) } driverMD := map[string]string{ cnpgServerNameKey: sourceApp, cnpgDestinationPathKey: "s3://bucket/" + sourceApp + "/", cnpgBackupNameKey: cnpgBkName, } if tc.recorded != "" { driverMD[cnpgMajorVersionKey] = tc.recorded } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: sourceApp}, StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, DriverMetadata: driverMD, }, Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, } rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, } if tc.toCopy { rj.Spec.TargetApplicationRef = &corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: copyApp} } app := newPostgresApp(targetApp, ns) app.Spec.Version = tc.targetVersion objs := []client.Object{backup, rj, strategy, app, &cnpgtypes.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: targetCluster}}, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ Namespace: ns, Name: targetCluster + "-1", Labels: map[string]string{cnpgClusterLabel: targetCluster}, }}, } if tc.cnpgMajor >= 0 { objs = append(objs, &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName}, Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: cnpgClusterNameForApp(sourceApp)}}, Status: cnpgtypes.BackupStatus{ Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003", MajorVersion: tc.cnpgMajor, }, }) } s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = cnpgtypes.AddToScheme(s) _ = postgresapp.AddToScheme(s) var deleted []string c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(objs...). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). WithInterceptorFuncs(interceptor.Funcs{ Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { deleted = append(deleted, fmt.Sprintf("%T %s", obj, obj.GetName())) return cl.Delete(ctx, obj, opts...) }, }). Build() r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)} live := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, live); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) } if _, err := r.reconcileCNPGRestore(ctx, live, backup); err != nil { t.Fatalf("reconcileCNPGRestore: %v", err) } got := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil { t.Fatalf("get RestoreJob after reconcile: %v", err) } patched := &postgresapp.Postgres{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: targetApp}, patched); err != nil { t.Fatalf("get target Postgres app: %v", err) } if !tc.wantRefused { if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("restore failed: %s", got.Status.Message) } if len(deleted) == 0 || !patched.Spec.Bootstrap.Enabled { t.Fatalf("restore did not reach the purge: deleted=%v bootstrap.enabled=%v", deleted, patched.Spec.Bootstrap.Enabled) } return } if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Errorf("expected phase Failed, got %q", got.Status.Phase) } if cond := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); cond == nil || cond.Reason != "MajorVersionMismatch" { t.Errorf("expected Ready=False reason MajorVersionMismatch, got %+v", cond) } for _, major := range []string{"PostgreSQL 16", "PostgreSQL 18"} { if !strings.Contains(got.Status.Message, major) { t.Errorf("message must name both majors, %q missing from %q", major, got.Status.Message) } } if len(deleted) != 0 { t.Errorf("a refused restore must delete nothing, deleted %v", deleted) } if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: targetCluster}, &cnpgtypes.Cluster{}); err != nil { t.Errorf("target Cluster must survive: %v", err) } if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: targetCluster + "-1"}, &corev1.PersistentVolumeClaim{}); err != nil { t.Errorf("target PVC must survive: %v", err) } if patched.Spec.Bootstrap.Enabled { t.Errorf("a refused restore must not patch the target app") } }) } } func (f *cnpgRolloutFixture) setStrategyDestination(destination, endpoint, secret string) { f.t.Helper() strategy := &strategyv1alpha1.CNPG{} if err := f.c.Get(context.Background(), client.ObjectKey{Name: "cnpg-strategy"}, strategy); err != nil { f.t.Fatalf("get strategy: %v", err) } strategy.Spec.Template.BarmanObjectStore.DestinationPath = destination strategy.Spec.Template.BarmanObjectStore.EndpointURL = endpoint strategy.Spec.Template.BarmanObjectStore.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{SecretRef: corev1.LocalObjectReference{Name: secret}} if err := f.c.Update(context.Background(), strategy); err != nil { f.t.Fatalf("update strategy: %v", err) } } // completeOwnBackup applies edit, when given, to the job's cnpg.io/Backup and // marks it completed. func (f *cnpgRolloutFixture) completeOwnBackup(edit func(b *cnpgtypes.Backup)) { f.t.Helper() backups := &cnpgtypes.BackupList{} if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil { f.t.Fatalf("list cnpg.io/Backups: %v", err) } for i := range backups.Items { b := &backups.Items[i] if b.Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" { continue } if edit != nil { edit(b) } b.Status.Phase = cnpgBackupPhaseComplete if err := f.c.Update(context.Background(), b); err != nil { f.t.Fatalf("complete cnpg.io/Backup: %v", err) } return } f.t.Fatalf("the job has no cnpg.io/Backup") } func (f *cnpgRolloutFixture) artifact() *backupsv1alpha1.Backup { f.t.Helper() artifact := &backupsv1alpha1.Backup{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, artifact); err != nil { if apierrors.IsNotFound(err) { return nil } f.t.Fatalf("get Backup artifact: %v", err) } return artifact } // The destination is written by the pass that creates the cnpg.io/Backup and // not again while it runs, so the artifact has to name what that pass wrote: // a strategy edited while the backup runs does not reach the backup, and must // not reach its artifact either. The BackupJob, which a tenant can edit, // carries no part of it. func TestReconcileCNPG_ArtifactNamesTheDestinationItsBackupWasCreatedWith(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.setStrategyDestination("s3://bucket/", "https://s3.one", "creds-one") f.pass() f.settleDestination() j, backups := f.pass() if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups once the destination settled, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) } if got := f.liveDestination(); got != "s3://bucket/" { t.Fatalf("destination = %q, want s3://bucket/", got) } want := `{"destinationPath":"s3://bucket/","endpointURL":"https://s3.one","s3SecretRef":"creds-one","serverName":"` + rolloutCluster + `"}` if got := backups[0].Annotations[cnpgWrittenDestinationAnnotation]; got != want { t.Fatalf("%s on the created cnpg.io/Backup = %q, want %q", cnpgWrittenDestinationAnnotation, got, want) } f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") j.Annotations = map[string]string{cnpgWrittenDestinationAnnotation: `{"destinationPath":"s3://tenant/","endpointURL":"https://tenant","s3SecretRef":"creds-one"}`} if err := f.c.Update(context.Background(), j); err != nil { t.Fatalf("annotate BackupJob: %v", err) } f.completeOwnBackup(nil) j, _ = f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) } artifact := f.artifact() if artifact == nil { t.Fatalf("no Backup artifact") } md := artifact.Spec.DriverMetadata if md[cnpgDestinationPathKey] != "s3://bucket/" || md[cnpgEndpointURLKey] != "https://s3.one" || md[cnpgS3SecretRefKey] != "creds-one" { t.Fatalf("artifact driver metadata = %v, want the destination written with the Backup (s3://bucket/, https://s3.one, creds-one)", md) } } // The serverName is the archive prefix under the destination, so the record // carries it too: a restore that re-renders the Cluster between the backup // and its artifact gives the live Cluster another one. A record written // before it carried a serverName falls back to the live Cluster. func TestReconcileCNPG_ArtifactNamesTheServerNameItsBackupWasCreatedWith(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") _, backups := f.pass() want := `{"destinationPath":"s3://bucket/","serverName":"` + rolloutCluster + `"}` if len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != want { t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording %s", backups, want) } f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, "restored-server"))) f.completeOwnBackup(nil) j, _ := f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) } artifact := f.artifact() if got := artifact.Spec.DriverMetadata[cnpgServerNameKey]; got != rolloutCluster { t.Errorf("artifact serverName = %q, want %q recorded with the Backup", got, rolloutCluster) } if !strings.HasPrefix(artifact.Status.Artifact.URI, "cnpg://"+rolloutCluster+"/") { t.Errorf("artifact URI = %q, want it under %s", artifact.Status.Artifact.URI, rolloutCluster) } } func TestReconcileCNPG_ArtifactOfARecordWithoutServerName(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") f.pass() f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, "live-server"))) f.completeOwnBackup(func(b *cnpgtypes.Backup) { b.Annotations[cnpgWrittenDestinationAnnotation] = `{"destinationPath":"s3://bucket/"}` }) j, _ := f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) } if got := f.artifact().Spec.DriverMetadata[cnpgServerNameKey]; got != "live-server" { t.Errorf("artifact serverName = %q, want the live Cluster's live-server", got) } } // A strategy without credentials records none, and the artifact names none. func TestReconcileCNPG_ArtifactOfARecordWithoutCredentials(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") if _, backups := f.pass(); len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != `{"destinationPath":"s3://bucket/","serverName":"`+rolloutCluster+`"}` { t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording only s3://bucket/", backups) } f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") f.completeOwnBackup(nil) j, _ := f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) } md := f.artifact().Spec.DriverMetadata if _, ok := md[cnpgS3SecretRefKey]; ok || md[cnpgDestinationPathKey] != "s3://bucket/" || md[cnpgEndpointURLKey] != "" { t.Fatalf("artifact driver metadata = %v, want s3://bucket/ with no endpoint and no credentials", md) } } // A record that names no usable destination cannot say where the backup is, // so no artifact is created rather than one pointing at the strategy as it // reads now. func TestReconcileCNPG_ArtifactRefusedForAnUnusableDestinationRecord(t *testing.T) { for name, value := range map[string]string{ "empty": "", "unreadable": "{not json", "not an object": `"s3://bucket/"`, "null": "null", "no fields": "{}", "empty destination": `{"destinationPath":"","endpointURL":"https://s3.one"}`, // Decoding sets the destination path and skips the mistyped // endpoint, which would leave the artifact on the default one. "mistyped endpoint": `{"destinationPath":"s3://bucket/","endpointURL":7}`, } { t.Run(name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") if _, backups := f.pass(); len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) } f.completeOwnBackup(func(b *cnpgtypes.Backup) { if b.Annotations == nil { b.Annotations = map[string]string{} } b.Annotations[cnpgWrittenDestinationAnnotation] = value }) j, _ := f.pass() if f.artifact() != nil { t.Fatalf("an artifact was created from the record %q", value) } if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgWrittenDestinationAnnotation) { t.Fatalf("phase = %q (%s), want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgWrittenDestinationAnnotation) } }) } } // A cnpg.io/Backup created by a controller that recorded nothing keeps // getting its artifact from the strategy. func TestReconcileCNPG_ArtifactOfABackupWithoutARecord(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") f.seedCNPGBackup("bj-legacy", "bj", rolloutCluster, cnpgBackupPhaseComplete) j, _ := f.pass() if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) } artifact := f.artifact() if artifact == nil || artifact.Spec.DriverMetadata[cnpgDestinationPathKey] != "s3://bucket/" { t.Fatalf("artifact = %+v, want one built from the strategy", artifact) } } // cacheLaggingBehind makes the reconciler's cached client miss the named // cnpg.io/Backups, as an informer does right after they are created, while // its uncached reader sees them. func (f *cnpgRolloutFixture) cacheLaggingBehind(hidden ...string) { f.r.APIReader = f.c f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { if err := cl.List(ctx, list, opts...); err != nil { return err } if backups, ok := list.(*cnpgtypes.BackupList); ok { backups.Items = slices.DeleteFunc(backups.Items, func(b cnpgtypes.Backup) bool { return slices.Contains(hidden, b.Name) }) } return nil }, }) } // Reconciles run one at a time, so a Backup created by one pass exists by // the next; the queue has to see it even when the cache does not yet, or // the next job rewrites the destination under it and the record on that // Backup no longer names where its WAL goes. func TestReconcileCNPG_QueueSeesABackupTheCacheHasNotCaught(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") f.cacheLaggingBehind("bj-other-cnpg") j, backups := f.pass() if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" { t.Fatalf("Ready = %+v, want WaitingForOtherBackup", ready) } if got := f.liveDestination(); got != "s3://other/" { t.Fatalf("destination = %q, rewritten under the other job's Backup", got) } if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups, want only the other job's", len(backups)) } } // The job's own Backup, missing from the cache right after it was created, // is neither created again nor written under. func TestReconcileCNPG_OwnBackupTheCacheHasNotCaught(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") f.cacheLaggingBehind("bj-cnpg") _, backups := f.pass() if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups, want the job's one", len(backups)) } if got := f.liveDestination(); got != "s3://other/" { t.Fatalf("destination = %q, rewritten under the job's own Backup", got) } } // failingBackupLists makes the uncached reader fail the cnpg.io/Backup lists // for which fail, given whether the list selects only the job's own Backup // (it would not find another job's) and how many such lists came before, // returns true. func (f *cnpgRolloutFixture) failingBackupLists(fail func(own bool, n int) bool) { counts := map[bool]int{} anotherJobs := labels.Set{ backupsv1alpha1.OwningJobNameLabel: "bj-other", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS, cnpgClusterLabel: rolloutCluster, } f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*cnpgtypes.BackupList); ok { o := &client.ListOptions{} o.ApplyOptions(opts) own := o.LabelSelector != nil && !o.LabelSelector.Matches(anotherJobs) counts[own]++ if fail(own, counts[own]) { return apierrors.NewServiceUnavailable("apiserver unavailable") } } return cl.List(ctx, list, opts...) }, }) } // A failed apiserver read of the Backups is retried, not taken as the job // failing: nothing has been written yet that the retry would not redo. func TestReconcileCNPG_BackupListErrorIsReturned(t *testing.T) { for name, own := range map[string]bool{"own Backup": true, "queue": false} { t.Run(name, func(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") f.failingBackupLists(func(o bool, _ int) bool { return o == own }) j := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { t.Fatalf("get BackupJob: %v", err) } if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { t.Fatalf("reconcileCNPG swallowed the Backup list error") } got := &backupsv1alpha1.BackupJob{} if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, got); err != nil { t.Fatalf("get BackupJob: %v", err) } if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("BackupJob failed on a Backup list error: %s", got.Status.Message) } if dest := f.liveDestination(); dest != "s3://other/" { t.Fatalf("destination = %q, written although the Backups could not be read", dest) } }) } } // The pass that writes the destination looks its own Backup up once, at the // start: a second lookup before the create could only fail, after the write, // and fail the job with it. func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") f.failingBackupLists(func(own bool, n int) bool { return own && n > 1 }) j, backups := f.pass() if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("BackupJob failed: %s", j.Status.Message) } if len(backups) != 1 { t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups)) } }