From a130813f4454713d0f6bb2384bed13c1b392cad6 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 12 Nov 2026 18:17:51 +0300 Subject: [PATCH 1/2] fix(postgres): restore from an archive with the access it was written with A CNPG restore took the destination path and endpoint of the source from the Backup artifact but the credentials Secret, its key names and the endpoint CA from the strategy as it reads at restore time. Once the strategy pointed at another bucket or endpoint, the restore combined the recorded location with access settings meant for a different one, and recovery failed after an in-place restore had already purged the target Cluster. The destination record on the cnpg.io/Backup now also carries the key names and the CA, and the artifact copies them. A restore uses them once the strategy points elsewhere and their Secrets still hold the keys; otherwise it keeps the strategy's access, as before, which credentials wider than one bucket can still read the old archive with. While the strategy points at the same endpoint and bucket its own access is kept: WAL archiving keeps proving it against that archive, and it carries any rotation since the backup, after which the recorded Secret may already be revoked. The size probe reads the recorded access too. An unreadable record fails the restore before the purge instead of falling back. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../cnpg_archive_access_test.go | 536 ++++++++++++++++++ .../cnpgstrategy_controller.go | 200 ++++++- .../cnpgstrategy_controller_test.go | 6 +- 3 files changed, 717 insertions(+), 25 deletions(-) create mode 100644 internal/backupcontroller/cnpg_archive_access_test.go diff --git a/internal/backupcontroller/cnpg_archive_access_test.go b/internal/backupcontroller/cnpg_archive_access_test.go new file mode 100644 index 000000000..202f22b55 --- /dev/null +++ b/internal/backupcontroller/cnpg_archive_access_test.go @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes" + "github.com/cozystack/cozystack/internal/backupcontroller/postgresapp" +) + +// archiveStoreA is the store a backup is written to; archiveStoreB is the +// same strategy after its credentials Secret, key names and CA were moved. +func archiveStoreA() strategyv1alpha1.BarmanObjectStoreTemplate { + return strategyv1alpha1.BarmanObjectStoreTemplate{ + DestinationPath: "s3://bkt-a/tenant/pg/", + EndpointURL: "https://s3-a.example:8443", + S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ + SecretRef: corev1.LocalObjectReference{Name: "creds-a"}, + AccessKeyIDKey: "id-a", + SecretAccessKeyKey: "key-a", + }, + EndpointCA: &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca-a"}, Key: "ca-a.crt"}, + } +} + +func archiveStoreB() strategyv1alpha1.BarmanObjectStoreTemplate { + return strategyv1alpha1.BarmanObjectStoreTemplate{ + DestinationPath: "s3://bkt-b/tenant/pg/", + EndpointURL: "https://s3-b.example:8443", + S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ + SecretRef: corev1.LocalObjectReference{Name: "creds-b"}, + AccessKeyIDKey: "id-b", + SecretAccessKeyKey: "key-b", + }, + EndpointCA: &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca-b"}, Key: "ca-b.crt"}, + } +} + +// artifactDriverMetadata takes a backup written with store through the +// path a real one follows: the record annotated on the cnpg.io/Backup at +// creation, read back when it completes, and copied into the artifact. +func artifactDriverMetadata(t *testing.T, store strategyv1alpha1.BarmanObjectStoreTemplate) map[string]string { + t.Helper() + written := cnpgDestinationOf(&strategyv1alpha1.CNPGTemplate{BarmanObjectStore: store}) + raw, err := json.Marshal(written) + if err != nil { + t.Fatalf("marshal the record: %v", err) + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{ + Name: "cnpg-bk", + Annotations: map[string]string{cnpgWrittenDestinationAnnotation: string(raw)}, + }} + moved := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: archiveStoreB()} + destination, err := cnpgArtifactDestination(cnpgBk, moved) + if err != nil { + t.Fatalf("cnpgArtifactDestination: %v", err) + } + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + r := &BackupJobReconciler{Client: newCNPGStrategyTestClient(t)} + artifact, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", destination, moved, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + return artifact.Spec.DriverMetadata +} + +// A restore reaches the archive with the access settings the backup was +// written with, whatever the strategy names by the time of the restore. +func TestCNPGRestoreArchiveAccess_RecordedAccessWins(t *testing.T) { + a := archiveStoreA() + b := archiveStoreB() + creds, ca, err := cnpgRestoreArchiveAccess(artifactDriverMetadata(t, a), &b) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if !reflect.DeepEqual(creds, a.S3Credentials) { + t.Errorf("credentials = %+v, want the recorded %+v", creds, a.S3Credentials) + } + if !reflect.DeepEqual(ca, a.EndpointCA) { + t.Errorf("endpoint CA = %+v, want the recorded %+v", ca, a.EndpointCA) + } +} + +// A backup written without credentials or a CA is restored without them, +// not with the ones the strategy gained since: they belong to another +// location. +func TestCNPGRestoreArchiveAccess_RecordedAbsenceWins(t *testing.T) { + a := archiveStoreA() + a.S3Credentials = nil + a.EndpointCA = nil + b := archiveStoreB() + creds, ca, err := cnpgRestoreArchiveAccess(artifactDriverMetadata(t, a), &b) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if creds != nil || ca != nil { + t.Errorf("credentials = %+v, endpoint CA = %+v, want neither: the backup was written without them", creds, ca) + } +} + +// Default key names are recorded as the strategy rendered them, empty, so +// the chart keeps applying its defaults rather than the strategy's new ones. +func TestCNPGRestoreArchiveAccess_RecordedDefaultKeysStayDefault(t *testing.T) { + a := archiveStoreA() + a.S3Credentials.AccessKeyIDKey = "" + a.S3Credentials.SecretAccessKeyKey = "" + a.EndpointCA.Key = "" + b := archiveStoreB() + creds, ca, err := cnpgRestoreArchiveAccess(artifactDriverMetadata(t, a), &b) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + want := &strategyv1alpha1.S3CredentialsTemplate{SecretRef: corev1.LocalObjectReference{Name: "creds-a"}} + if !reflect.DeepEqual(creds, want) { + t.Errorf("credentials = %+v, want %+v", creds, want) + } + if ca == nil || ca.SecretRef.Name != "ca-a" || ca.Key != "" { + t.Errorf("endpoint CA = %+v, want ca-a with the default key", ca) + } +} + +// An artifact from a controller that recorded only the Secret keeps that +// Secret and takes the key names and the CA it never recorded from the +// strategy. +func TestCNPGRestoreArchiveAccess_LegacyArtifact(t *testing.T) { + b := archiveStoreB() + for _, tc := range []struct { + name string + md map[string]string + wantSecret string + }{ + {name: "with a recorded Secret", md: map[string]string{cnpgS3SecretRefKey: "creds-a"}, wantSecret: "creds-a"}, + {name: "without one", md: map[string]string{}, wantSecret: "creds-b"}, + } { + t.Run(tc.name, func(t *testing.T) { + creds, ca, err := cnpgRestoreArchiveAccess(tc.md, &b) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if creds == nil || creds.SecretRef.Name != tc.wantSecret || creds.AccessKeyIDKey != "id-b" || creds.SecretAccessKeyKey != "key-b" { + t.Errorf("credentials = %+v, want Secret %s with the strategy's key names", creds, tc.wantSecret) + } + if !reflect.DeepEqual(ca, b.EndpointCA) { + t.Errorf("endpoint CA = %+v, want the strategy's %+v", ca, b.EndpointCA) + } + }) + } +} + +// The legacy fallback never writes through to the strategy it was given. +func TestCNPGRestoreArchiveAccess_LegacyArtifactLeavesTheStrategyAlone(t *testing.T) { + b := archiveStoreB() + if _, _, err := cnpgRestoreArchiveAccess(map[string]string{cnpgS3SecretRefKey: "creds-a"}, &b); err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if b.S3Credentials.SecretRef.Name != "creds-b" { + t.Errorf("the rendered strategy now names Secret %q", b.S3Credentials.SecretRef.Name) + } +} + +func TestCNPGRestoreArchiveAccess_UnreadableRecordFails(t *testing.T) { + b := archiveStoreB() + if _, _, err := cnpgRestoreArchiveAccess(map[string]string{cnpgArchiveAccessKey: "{"}, &b); err == nil { + t.Fatalf("an unreadable record was accepted; the restore would fall back to a strategy that may name another location") + } +} + +// A record written before it carried the access settings leaves the +// artifact without them, so a restore treats it as legacy rather than as +// a backup written without credentials or a CA. +func TestCreateCNPGBackupArtifact_RecordWithoutAccessRecordsNone(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + secret := "creds-a" + written := cnpgDestination{DestinationPath: "s3://bkt-a/tenant/pg/", S3SecretRef: &secret} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: archiveStoreB()} + r := &BackupJobReconciler{Client: newCNPGStrategyTestClient(t)} + artifact, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if v, ok := artifact.Spec.DriverMetadata[cnpgArchiveAccessKey]; ok { + t.Errorf("driverMetadata[%s] = %q, want it absent for a record that carries no access settings", cnpgArchiveAccessKey, v) + } +} + +// The size probe reads the recorded Secret under the recorded key names. +func TestCNPGProbeStore_UsesTheRecordedAccess(t *testing.T) { + a := archiveStoreA() + b := archiveStoreB() + written := cnpgDestinationOf(&strategyv1alpha1.CNPGTemplate{BarmanObjectStore: a}) + store := cnpgProbeStore(&b, written) + if !reflect.DeepEqual(store.S3Credentials, a.S3Credentials) { + t.Errorf("probe credentials = %+v, want the recorded %+v", store.S3Credentials, a.S3Credentials) + } + if !reflect.DeepEqual(store.EndpointCA, a.EndpointCA) { + t.Errorf("probe endpoint CA = %+v, want the recorded %+v", store.EndpointCA, a.EndpointCA) + } + if b.S3Credentials.SecretRef.Name != "creds-b" || b.EndpointCA.SecretRef.Name != "ca-b" { + t.Errorf("the probe wrote through to the rendered strategy: %+v", b) + } +} + +// TestReconcileCNPGRestore_PatchesTheRecordedAccess drives a restore up to +// the purge after the strategy moved to another bucket with another +// credentials Secret, key names and CA, and reads back what the target app +// is patched with. +func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { + const ( + ns = "tenant" + appName = "pg" + ) + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + ctx := context.Background() + + secret := func(name string, keys ...string) *corev1.Secret { + data := map[string][]byte{} + for _, k := range keys { + data[k] = []byte("x") + } + return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}, Data: data} + } + emptyKey := func(s *corev1.Secret, key string) *corev1.Secret { + s.Data[key] = nil + return s + } + recordedSecrets := func() []client.Object { + return []client.Object{secret("creds-a", "id-a", "key-a"), secret("ca-a", "ca-a.crt")} + } + + type outcome struct { + app *postgresapp.Postgres + rj *backupsv1alpha1.RestoreJob + err error + events []string + } + restore := func(t *testing.T, md map[string]string, getErr error, objs ...client.Object) outcome { + t.Helper() + snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil, "") + if err != nil { + t.Fatalf("marshal snapshot: %v", err) + } + md[cnpgServerNameKey] = "srv" + md[cnpgBackupNameKey] = "cnpg-bk" + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, + DriverMetadata: md, + }, + Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, + } + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, + Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{BarmanObjectStore: archiveStoreB()}}, + } + sa := metav1.NewTime(time.Now()) + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + c := newCNPGStrategyTestClient(t, append([]client.Object{backup, rj, strategy, newPostgresApp(appName, ns)}, objs...)...) + var cl client.Client = c + if getErr != nil { + cl = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, wc client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok { + return getErr + } + return wc.Get(ctx, key, obj, opts...) + }, + }) + } + recorder := record.NewFakeRecorder(10) + r := &RestoreJobReconciler{Client: cl, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: recorder} + live := &backupsv1alpha1.RestoreJob{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, live); err != nil { + t.Fatalf("get seeded RestoreJob: %v", err) + } + _, reconcileErr := r.reconcileCNPGRestore(ctx, live, backup) + app := &postgresapp.Postgres{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, app); err != nil { + t.Fatalf("get Postgres app: %v", err) + } + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, live); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + var events []string + for len(recorder.Events) > 0 { + events = append(events, <-recorder.Events) + } + return outcome{app: app, rj: live, err: reconcileErr, events: events} + } + recordedCreds := postgresapp.S3CredentialsSecret{Name: "creds-a", AccessKeyIDKey: "id-a", SecretAccessKeyKey: "key-a"} + recordedCA := postgresapp.EndpointCA{Name: "ca-a", Key: "ca-a.crt"} + strategyCreds := postgresapp.S3CredentialsSecret{Name: "creds-b", AccessKeyIDKey: "id-b", SecretAccessKeyKey: "key-b"} + strategyCA := postgresapp.EndpointCA{Name: "ca-b", Key: "ca-b.crt"} + patched := func(t *testing.T, o outcome, wantCreds postgresapp.S3CredentialsSecret, wantCA postgresapp.EndpointCA) { + t.Helper() + if o.err != nil { + t.Fatalf("reconcileCNPGRestore: %v", o.err) + } + if !o.app.Spec.Bootstrap.Enabled { + t.Fatalf("the reconcile did not reach the restore patch") + } + if o.app.Spec.Backup.S3CredentialsSecret != wantCreds { + t.Errorf("s3CredentialsSecret = %+v, want %+v", o.app.Spec.Backup.S3CredentialsSecret, wantCreds) + } + if o.app.Spec.Backup.EndpointCA != wantCA { + t.Errorf("endpointCA = %+v, want %+v", o.app.Spec.Backup.EndpointCA, wantCA) + } + } + + t.Run("the recorded access is patched in", func(t *testing.T) { + o := restore(t, artifactDriverMetadata(t, archiveStoreA()), nil, recordedSecrets()...) + patched(t, o, recordedCreds, recordedCA) + for _, ev := range o.events { + if strings.Contains(ev, "RecordedArchiveAccessUnavailable") { + t.Errorf("event %q, but the recorded access was used", ev) + } + } + }) + + // Without its Secrets the recorded access cannot work; the strategy's + // is what the restore used before the record existed, and may still + // read the old bucket. + for name, objs := range map[string][]client.Object{ + "the recorded Secret is gone": {secret("ca-a", "ca-a.crt")}, + "the recorded Secret lacks a key": {secret("creds-a", "id-a"), secret("ca-a", "ca-a.crt")}, + "the recorded Secret has an empty key": {emptyKey(secret("creds-a", "id-a", "key-a"), "key-a"), secret("ca-a", "ca-a.crt")}, + "the recorded CA Secret is gone": {secret("creds-a", "id-a", "key-a")}, + "the recorded CA Secret lacks its key": {secret("creds-a", "id-a", "key-a"), secret("ca-a", "other.crt")}, + } { + t.Run(name+": the strategy's access is patched in", func(t *testing.T) { + o := restore(t, artifactDriverMetadata(t, archiveStoreA()), nil, objs...) + patched(t, o, strategyCreds, strategyCA) + found := false + for _, ev := range o.events { + found = found || strings.Contains(ev, "RecordedArchiveAccessUnavailable") + } + if !found { + t.Errorf("events = %v, want one saying the recorded access was not used", o.events) + } + }) + } + + t.Run("a recorded CA alone is checked too", func(t *testing.T) { + a := archiveStoreA() + a.S3Credentials = archiveStoreB().S3Credentials + o := restore(t, artifactDriverMetadata(t, a), nil, secret("creds-b", "id-b", "key-b")) + patched(t, o, strategyCreds, strategyCA) + found := false + for _, ev := range o.events { + found = found || strings.Contains(ev, "RecordedArchiveAccessUnavailable") + } + if !found { + t.Errorf("events = %v, want one saying the recorded access was not used", o.events) + } + }) + + // A reference with no Secret name is no reference: the backup ran + // without it, and the restore patch drops it. + t.Run("an empty recorded CA name is no CA", func(t *testing.T) { + a := archiveStoreA() + a.EndpointCA = &strategyv1alpha1.EndpointCARef{} + o := restore(t, artifactDriverMetadata(t, a), nil, secret("creds-a", "id-a", "key-a")) + patched(t, o, recordedCreds, postgresapp.EndpointCA{}) + }) + t.Run("an empty recorded Secret name is no credentials", func(t *testing.T) { + a := archiveStoreA() + a.S3Credentials.SecretRef.Name = "" + o := restore(t, artifactDriverMetadata(t, a), nil, secret("ca-a", "ca-a.crt")) + patched(t, o, postgresapp.S3CredentialsSecret{}, recordedCA) + }) + + t.Run("recorded credentials alone are checked too", func(t *testing.T) { + a := archiveStoreA() + a.EndpointCA = archiveStoreB().EndpointCA + o := restore(t, artifactDriverMetadata(t, a), nil, secret("ca-b", "ca-b.crt")) + patched(t, o, strategyCreds, strategyCA) + found := false + for _, ev := range o.events { + found = found || strings.Contains(ev, "RecordedArchiveAccessUnavailable") + } + if !found { + t.Errorf("events = %v, want one saying the recorded access was not used", o.events) + } + }) + + // The strategy's own access is what a restore of an archive that has not + // moved uses; nothing recorded is passed over, so nothing is reported, + // whether or not its Secrets exist. + t.Run("the same archive reports nothing", func(t *testing.T) { + o := restore(t, artifactDriverMetadata(t, archiveStoreB()), nil) + patched(t, o, strategyCreds, strategyCA) + for _, ev := range o.events { + if strings.Contains(ev, "RecordedArchiveAccessUnavailable") { + t.Errorf("event %q for an archive the strategy still points at", ev) + } + } + }) + + t.Run("default key names are looked up under their defaults", func(t *testing.T) { + a := archiveStoreA() + a.S3Credentials.AccessKeyIDKey, a.S3Credentials.SecretAccessKeyKey, a.EndpointCA.Key = "", "", "" + o := restore(t, artifactDriverMetadata(t, a), nil, + secret("creds-a", defaultS3AccessKeyIDKey, defaultS3SecretAccessKeyKey), secret("ca-a", defaultEndpointCAKey)) + patched(t, o, postgresapp.S3CredentialsSecret{Name: "creds-a"}, postgresapp.EndpointCA{Name: "ca-a"}) + }) + + t.Run("a Secret read error is returned before the patch", func(t *testing.T) { + o := restore(t, artifactDriverMetadata(t, archiveStoreA()), apierrors.NewServiceUnavailable("apiserver unavailable"), recordedSecrets()...) + if o.err == nil { + t.Errorf("reconcileCNPGRestore swallowed the Secret read error") + } + if o.app.Spec.Bootstrap.Enabled { + t.Errorf("the target app was patched although the recorded access could not be checked") + } + }) + + t.Run("an unreadable record fails the restore before the purge", func(t *testing.T) { + o := restore(t, map[string]string{cnpgDestinationPathKey: "s3://bkt-a/tenant/pg/", cnpgArchiveAccessKey: "{"}, nil) + if o.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("phase = %q, want Failed", o.rj.Status.Phase) + } + if o.app.Spec.Bootstrap.Enabled { + t.Errorf("the target app was patched for recovery from a record that could not be read") + } + }) +} + +// Access to an archive that has not moved is rotated in the strategy: the +// Secret renamed, the keys renamed, the CA reissued, the old ones revoked. +// The strategy's access is the one WAL archiving keeps proving against +// that archive, so a restore takes it rather than the recorded one. +func TestCNPGRestoreArchiveAccess_RotatedAccessToTheSameArchive(t *testing.T) { + rotated := archiveStoreA() + rotated.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{ + SecretRef: corev1.LocalObjectReference{Name: "creds-a-rotated"}, + AccessKeyIDKey: "id-rotated", + SecretAccessKeyKey: "key-rotated", + } + rotated.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca-a-rotated"}} + for name, edit := range map[string]func(*strategyv1alpha1.BarmanObjectStoreTemplate){ + "same prefix": func(*strategyv1alpha1.BarmanObjectStoreTemplate) {}, + "another prefix": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.DestinationPath = "s3://bkt-a/tenant/pg-copy/" }, + "endpoint trailing slash": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.EndpointURL += "/" }, + } { + t.Run(name, func(t *testing.T) { + store := rotated + store.S3Credentials = rotated.S3Credentials.DeepCopy() + store.EndpointCA = rotated.EndpointCA.DeepCopy() + edit(&store) + creds, ca, err := cnpgRestoreArchiveAccess(artifactDriverMetadata(t, archiveStoreA()), &store) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if !reflect.DeepEqual(creds, store.S3Credentials) { + t.Errorf("credentials = %+v, want the strategy's rotated %+v", creds, store.S3Credentials) + } + if !reflect.DeepEqual(ca, store.EndpointCA) { + t.Errorf("endpoint CA = %+v, want the strategy's rotated %+v", ca, store.EndpointCA) + } + }) + } +} + +// The same holds for an artifact that recorded only the Secret. +func TestCNPGRestoreArchiveAccess_LegacyArtifactOfTheSameArchive(t *testing.T) { + a := archiveStoreA() + md := map[string]string{ + cnpgDestinationPathKey: a.DestinationPath, + cnpgEndpointURLKey: a.EndpointURL, + cnpgS3SecretRefKey: "creds-a-revoked", + } + creds, _, err := cnpgRestoreArchiveAccess(md, &a) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if !reflect.DeepEqual(creds, a.S3Credentials) { + t.Errorf("credentials = %+v, want the strategy's %+v", creds, a.S3Credentials) + } +} + +// The location is the endpoint and the bucket; either one moving makes the +// strategy's access belong to another archive. +func TestCNPGRestoreArchiveAccess_MovedArchiveKeepsTheRecordedAccess(t *testing.T) { + a := archiveStoreA() + for name, edit := range map[string]func(*strategyv1alpha1.BarmanObjectStoreTemplate){ + "another bucket": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.DestinationPath = "s3://bkt-b/tenant/pg/" }, + "another endpoint": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.EndpointURL = "https://s3-b.example:8443" }, + "not s3": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.DestinationPath = "gs://bkt-a/tenant/pg/" }, + } { + t.Run(name, func(t *testing.T) { + store := archiveStoreB() + store.DestinationPath, store.EndpointURL = a.DestinationPath, a.EndpointURL + edit(&store) + creds, ca, err := cnpgRestoreArchiveAccess(artifactDriverMetadata(t, a), &store) + if err != nil { + t.Fatalf("cnpgRestoreArchiveAccess: %v", err) + } + if !reflect.DeepEqual(creds, a.S3Credentials) || !reflect.DeepEqual(ca, a.EndpointCA) { + t.Errorf("credentials = %+v, endpoint CA = %+v, want the recorded ones", creds, ca) + } + }) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index e43deff33..c702afa88 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -69,6 +69,7 @@ const ( cnpgClusterNameKey = "cnpg.io/cluster-name" cnpgS3SecretRefKey = "cnpg.io/s3-secret-ref" cnpgMajorVersionKey = "cnpg.io/major-version" + cnpgArchiveAccessKey = "cnpg.io/archive-access" // Annotation the postgres chart renders on the Cluster from // bootstrap.newServerName. @@ -1233,27 +1234,159 @@ func (r *BackupJobReconciler) waitCNPGBackupJob(ctx context.Context, j *backupsv // cnpgDestination is the WAL archive destination recorded in // cnpgWrittenDestinationAnnotation. S3SecretRef is nil when the strategy -// names no credentials Secret. ServerName is empty in a record written -// before it carried one. +// names no credentials Secret. ServerName is empty, and Access nil, in a +// record written before it carried them. type cnpgDestination struct { - DestinationPath string `json:"destinationPath"` - EndpointURL string `json:"endpointURL,omitempty"` - S3SecretRef *string `json:"s3SecretRef,omitempty"` - ServerName string `json:"serverName,omitempty"` + DestinationPath string `json:"destinationPath"` + EndpointURL string `json:"endpointURL,omitempty"` + S3SecretRef *string `json:"s3SecretRef,omitempty"` + ServerName string `json:"serverName,omitempty"` + Access *cnpgArchiveAccess `json:"access,omitempty"` +} + +// cnpgArchiveAccess is how the Secret in S3SecretRef and the endpoint CA +// were read when the archive was written. Empty key names are the chart's +// defaults; a nil EndpointCA is the system trust store. +type cnpgArchiveAccess struct { + AccessKeyIDKey string `json:"accessKeyIDKey,omitempty"` + SecretAccessKeyKey string `json:"secretAccessKeyKey,omitempty"` + EndpointCA *strategyv1alpha1.EndpointCARef `json:"endpointCA,omitempty"` } func cnpgDestinationOf(t *strategyv1alpha1.CNPGTemplate) cnpgDestination { d := cnpgDestination{ DestinationPath: t.BarmanObjectStore.DestinationPath, EndpointURL: t.BarmanObjectStore.EndpointURL, + Access: &cnpgArchiveAccess{}, } - if t.BarmanObjectStore.S3Credentials != nil { - name := t.BarmanObjectStore.S3Credentials.SecretRef.Name + if creds := t.BarmanObjectStore.S3Credentials; creds != nil { + name := creds.SecretRef.Name d.S3SecretRef = &name + d.Access.AccessKeyIDKey = creds.AccessKeyIDKey + d.Access.SecretAccessKeyKey = creds.SecretAccessKeyKey + } + if ca := t.BarmanObjectStore.EndpointCA; ca != nil { + d.Access.EndpointCA = ca.DeepCopy() } return d } +// archiveStore returns the credentials and the endpoint CA the archive +// was written with. Without a recorded Access the record names only the +// Secret, so the key names and the CA come from fallback. +func (d cnpgDestination) archiveStore(fallback *strategyv1alpha1.BarmanObjectStoreTemplate) (*strategyv1alpha1.S3CredentialsTemplate, *strategyv1alpha1.EndpointCARef) { + if d.Access != nil { + var creds *strategyv1alpha1.S3CredentialsTemplate + if d.S3SecretRef != nil { + creds = &strategyv1alpha1.S3CredentialsTemplate{ + SecretRef: corev1.LocalObjectReference{Name: *d.S3SecretRef}, + AccessKeyIDKey: d.Access.AccessKeyIDKey, + SecretAccessKeyKey: d.Access.SecretAccessKeyKey, + } + } + return creds, d.Access.EndpointCA.DeepCopy() + } + var creds *strategyv1alpha1.S3CredentialsTemplate + if d.S3SecretRef != nil { + creds = &strategyv1alpha1.S3CredentialsTemplate{} + if fallback.S3Credentials != nil { + creds = fallback.S3Credentials.DeepCopy() + } + creds.SecretRef.Name = *d.S3SecretRef + } + return creds, fallback.EndpointCA.DeepCopy() +} + +// cnpgRestoreArchiveAccess returns the credentials and the endpoint CA a +// restore reaches the archive of an artifact with. While the strategy +// rendered now still points at the archive's endpoint and bucket, they are +// the strategy's: WAL archiving keeps proving them against that archive, +// and they carry any rotation since the backup, after which the recorded +// ones may be revoked. Once the strategy points elsewhere, they are the +// ones recorded with the artifact; an artifact from a controller that +// recorded only the Secret, or nothing, takes the rest from the strategy. +// An unreadable record is an error rather than a fallback. +func cnpgRestoreArchiveAccess(md map[string]string, rendered *strategyv1alpha1.BarmanObjectStoreTemplate) (*strategyv1alpha1.S3CredentialsTemplate, *strategyv1alpha1.EndpointCARef, error) { + raw, recorded := md[cnpgArchiveAccessKey] + secret, hasSecret := md[cnpgS3SecretRefKey] + d := cnpgDestination{} + if recorded { + d.Access = &cnpgArchiveAccess{} + if err := json.Unmarshal([]byte(raw), d.Access); err != nil { + return nil, nil, fmt.Errorf("read driverMetadata[%s]: %w", cnpgArchiveAccessKey, err) + } + } + if (!recorded && !hasSecret) || cnpgSameArchive(md[cnpgDestinationPathKey], md[cnpgEndpointURLKey], rendered) { + return rendered.S3Credentials.DeepCopy(), rendered.EndpointCA.DeepCopy(), nil + } + if hasSecret { + d.S3SecretRef = &secret + } + creds, ca := d.archiveStore(rendered) + return creds, ca, nil +} + +// cnpgArchiveAccessMissing returns what of creds and ca is missing from +// namespace, a Secret or a key it is read under, or "" when nothing is. A +// reference without a Secret name is no reference, as the backup and the +// restore patch treat it, so there is nothing of it to miss. +func (r *RestoreJobReconciler) cnpgArchiveAccessMissing(ctx context.Context, namespace string, creds *strategyv1alpha1.S3CredentialsTemplate, ca *strategyv1alpha1.EndpointCARef) (string, error) { + type secretKeys struct { + name string + keys []string + } + var wanted []secretKeys + if creds != nil && creds.SecretRef.Name != "" { + accessKeyKey, secretKeyKey := creds.AccessKeyIDKey, creds.SecretAccessKeyKey + if accessKeyKey == "" { + accessKeyKey = defaultS3AccessKeyIDKey + } + if secretKeyKey == "" { + secretKeyKey = defaultS3SecretAccessKeyKey + } + wanted = append(wanted, secretKeys{creds.SecretRef.Name, []string{accessKeyKey, secretKeyKey}}) + } + if ca != nil && ca.SecretRef.Name != "" { + key := ca.Key + if key == "" { + key = defaultEndpointCAKey + } + wanted = append(wanted, secretKeys{ca.SecretRef.Name, []string{key}}) + } + for _, w := range wanted { + secret := &corev1.Secret{} + if err := r.Get(ctx, client.ObjectKey{Namespace: namespace, Name: w.name}, secret); err != nil { + if apierrors.IsNotFound(err) { + return fmt.Sprintf("Secret %s/%s does not exist", namespace, w.name), nil + } + return "", fmt.Errorf("read Secret %s/%s: %w", namespace, w.name, err) + } + for _, key := range w.keys { + if len(secret.Data[key]) == 0 { + return fmt.Sprintf("Secret %s/%s has no value under key %s", namespace, w.name, key), nil + } + } + } + return "", nil +} + +// cnpgSameArchive reports whether store reaches the S3 bucket at the +// endpoint an artifact recorded. The prefix within the bucket does not +// count: access is taken to be granted per bucket, as a Bucket grants it. +// A destination that is not an s3:// URL is never the same: what scopes +// its access is not known here. +func cnpgSameArchive(destinationPath, endpointURL string, store *strategyv1alpha1.BarmanObjectStoreTemplate) bool { + if strings.TrimSuffix(endpointURL, "/") != strings.TrimSuffix(store.EndpointURL, "/") { + return false + } + bucket, _, err := parseBarmanS3DestinationPath(destinationPath) + if err != nil { + return false + } + current, _, err := parseBarmanS3DestinationPath(store.DestinationPath) + return err == nil && bucket == current +} + // cnpgArtifactDestination returns the destination the artifact of b names: // the one recorded when b was created. A Backup without the record was // created by a controller that kept none, and falls back to the strategy @@ -1274,23 +1407,16 @@ func cnpgArtifactDestination(b *cnpgtypes.Backup, rendered *strategyv1alpha1.CNP return d, nil } -// cnpgProbeStore is the object store the size probe reads: the location -// and the credentials Secret recorded for the backup, with the Secret keys -// and the endpoint CA the record does not carry taken from the strategy. -// After a strategy edit those may not fit the recorded location; the probe -// then reports no size rather than measuring somewhere else. +// cnpgProbeStore is the object store the size probe reads: the one +// recorded for the backup. A record without Access takes the Secret keys +// and the endpoint CA from the strategy; after a strategy edit those may +// not fit the recorded location, and the probe then reports no size +// rather than measuring somewhere else. func cnpgProbeStore(rendered *strategyv1alpha1.BarmanObjectStoreTemplate, destination cnpgDestination) *strategyv1alpha1.BarmanObjectStoreTemplate { store := rendered.DeepCopy() store.DestinationPath = destination.DestinationPath store.EndpointURL = destination.EndpointURL - if destination.S3SecretRef == nil { - store.S3Credentials = nil - } else { - if store.S3Credentials == nil { - store.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{} - } - store.S3Credentials.SecretRef.Name = *destination.S3SecretRef - } + store.S3Credentials, store.EndpointCA = destination.archiveStore(rendered) return store } @@ -1328,6 +1454,13 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( if destination.S3SecretRef != nil { driverMD[cnpgS3SecretRefKey] = *destination.S3SecretRef } + if destination.Access != nil { + access, err := json.Marshal(destination.Access) + if err != nil { + return nil, fmt.Errorf("encode the archive access settings: %w", err) + } + driverMD[cnpgArchiveAccessKey] = string(access) + } // Copied rather than read back at restore time: barman retention deletes // the cnpg.io/Backup while the artifact can still be restored from. if cnpgBackup.Status.MajorVersion > 0 { @@ -1506,6 +1639,10 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore if err != nil { return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template CNPG strategy: %v", err)) } + sourceCredentials, sourceEndpointCA, err := cnpgRestoreArchiveAccess(backup.Spec.DriverMetadata, &rendered.BarmanObjectStore) + if err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("Backup %s/%s does not say how to reach its archive: %v", backup.Namespace, backup.Name, err)) + } clusterName := cnpgClusterNameForApp(target.AppName) @@ -1646,6 +1783,25 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore logger.Info("restore does not pin its base backup; the barman-cloud plugin picks one from the catalog", "backup", backup.Name, "recoveryTime", options.RecoveryTime) } + if !apiequality.Semantic.DeepEqual(sourceCredentials, rendered.BarmanObjectStore.S3Credentials) || + !apiequality.Semantic.DeepEqual(sourceEndpointCA, rendered.BarmanObjectStore.EndpointCA) { + missing, err := r.cnpgArchiveAccessMissing(ctx, target.Namespace, sourceCredentials, sourceEndpointCA) + if err != nil { + return ctrl.Result{}, err + } + // The recorded access cannot work without its Secrets. The + // strategy's is what a restore used before the record existed, and + // credentials wider than one bucket can still read the old one. + // Both settings fall back together: once part of what was + // recorded is gone, the strategy's pair is the one known to + // belong together. + if missing != "" { + r.Recorder.Eventf(restoreJob, corev1.EventTypeWarning, "RecordedArchiveAccessUnavailable", + "the access Backup %s/%s was written with is unusable (%s); reaching its archive with the strategy's instead", + backup.Namespace, backup.Name, missing) + sourceCredentials, sourceEndpointCA = rendered.BarmanObjectStore.S3Credentials, rendered.BarmanObjectStore.EndpointCA + } + } hrName := postgresAppPrefix + target.AppName if err := r.setCNPGRestoreHRSuspended(ctx, target.Namespace, hrName, true); err != nil { return ctrl.Result{}, err @@ -1659,7 +1815,7 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore logger.Info("restored WAL-archive serverName collided with the recovery source", "cluster", clusterName, "serverName", newServerName) } - if err := r.patchPostgresAppForRestore(ctx, targetApp, sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, options.RecoveryTime, backupID, rendered.BarmanObjectStore.S3Credentials, rendered.BarmanObjectStore.EndpointCA, sourceDatabases, sourceUsers); err != nil { + if err := r.patchPostgresAppForRestore(ctx, targetApp, sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, options.RecoveryTime, backupID, sourceCredentials, sourceEndpointCA, sourceDatabases, sourceUsers); err != nil { // Resume HR before terminal failure so an operator deleting // the failed RestoreJob does not leave the HR stuck. _ = r.setCNPGRestoreHRSuspended(ctx, target.Namespace, hrName, false) diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 5cd990f18..bba0b6b2b 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -6508,7 +6508,7 @@ func TestReconcileCNPG_ArtifactNamesTheDestinationItsBackupWasCreatedWith(t *tes if got := f.liveDestination(); got != "s3://bucket/" { t.Fatalf("destination = %q, want s3://bucket/", got) } - want := `{"destinationPath":"s3://bucket/","endpointURL":"https://s3.one","s3SecretRef":"creds-one","serverName":"` + rolloutCluster + `"}` + want := `{"destinationPath":"s3://bucket/","endpointURL":"https://s3.one","s3SecretRef":"creds-one","serverName":"` + rolloutCluster + `","access":{}}` if got := backups[0].Annotations[cnpgWrittenDestinationAnnotation]; got != want { t.Fatalf("%s on the created cnpg.io/Backup = %q, want %q", cnpgWrittenDestinationAnnotation, got, want) } @@ -6540,7 +6540,7 @@ func TestReconcileCNPG_ArtifactNamesTheDestinationItsBackupWasCreatedWith(t *tes func TestReconcileCNPG_ArtifactNamesTheServerNameItsBackupWasCreatedWith(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") _, backups := f.pass() - want := `{"destinationPath":"s3://bucket/","serverName":"` + rolloutCluster + `"}` + want := `{"destinationPath":"s3://bucket/","serverName":"` + rolloutCluster + `","access":{}}` if len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != want { t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording %s", backups, want) } @@ -6578,7 +6578,7 @@ func TestReconcileCNPG_ArtifactOfARecordWithoutServerName(t *testing.T) { // A strategy without credentials records none, and the artifact names none. func TestReconcileCNPG_ArtifactOfARecordWithoutCredentials(t *testing.T) { f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") - if _, backups := f.pass(); len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != `{"destinationPath":"s3://bucket/","serverName":"`+rolloutCluster+`"}` { + if _, backups := f.pass(); len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != `{"destinationPath":"s3://bucket/","serverName":"`+rolloutCluster+`","access":{}}` { t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording only s3://bucket/", backups) } f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") From f6f0be308f6a1d872ccd1a1d9ed777c809255e32 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 13 Nov 2026 17:02:17 +0300 Subject: [PATCH 2/2] fix(postgres): restore a moved archive recorded without credentials The postgres chart refuses to render bootstrap.enabled without a credentials Secret, so a restore that patched the target with the access recorded for an archive written without one failed at render time, after the target Cluster had already been purged. Before the access was recorded such a restore used the strategy's credentials, which can still read the old bucket. A recorded access without a credentials Secret now counts as unusable, like one whose Secrets are gone: the restore takes the strategy's access and says so in a Warning event. The tests that stop a restore on an unreadable record or a failed Secret read now also check that the target Cluster, its volumes and its HelmRelease are left alone. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../cnpg_archive_access_test.go | 85 ++++++++++++++++--- .../cnpgstrategy_controller.go | 15 ++-- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/internal/backupcontroller/cnpg_archive_access_test.go b/internal/backupcontroller/cnpg_archive_access_test.go index 202f22b55..5efdc4ff4 100644 --- a/internal/backupcontroller/cnpg_archive_access_test.go +++ b/internal/backupcontroller/cnpg_archive_access_test.go @@ -12,6 +12,9 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -102,10 +105,12 @@ func TestCNPGRestoreArchiveAccess_RecordedAccessWins(t *testing.T) { } } -// A backup written without credentials or a CA is restored without them, -// not with the ones the strategy gained since: they belong to another -// location. -func TestCNPGRestoreArchiveAccess_RecordedAbsenceWins(t *testing.T) { +// For a moved archive written without credentials or a CA the record says +// so: the strategy's, which belong to another location, are not reported +// as the recorded ones. What the restore uses instead is decided by +// reconcileCNPGRestore, which takes a record without credentials as +// unusable. +func TestCNPGRestoreArchiveAccess_ReportsARecordedAbsence(t *testing.T) { a := archiveStoreA() a.S3Credentials = nil a.EndpointCA = nil @@ -260,6 +265,8 @@ func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { rj *backupsv1alpha1.RestoreJob err error events []string + c client.Client + dyn dynamic.Interface } restore := func(t *testing.T, md map[string]string, getErr error, objs ...client.Object) outcome { t.Helper() @@ -301,7 +308,13 @@ func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { }) } recorder := record.NewFakeRecorder(10) - r := &RestoreJobReconciler{Client: cl, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: recorder} + hr := &unstructured.Unstructured{} + hr.SetGroupVersionKind(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}) + hr.SetNamespace(ns) + hr.SetName(postgresAppPrefix + appName) + _ = unstructured.SetNestedMap(hr.Object, map[string]any{}, "spec") + dyn := dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t), hr) + r := &RestoreJobReconciler{Client: cl, Interface: dyn, Recorder: recorder} live := &backupsv1alpha1.RestoreJob{} if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, live); err != nil { t.Fatalf("get seeded RestoreJob: %v", err) @@ -318,7 +331,7 @@ func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { for len(recorder.Events) > 0 { events = append(events, <-recorder.Events) } - return outcome{app: app, rj: live, err: reconcileErr, events: events} + return outcome{app: app, rj: live, err: reconcileErr, events: events, c: c, dyn: dyn} } recordedCreds := postgresapp.S3CredentialsSecret{Name: "creds-a", AccessKeyIDKey: "id-a", SecretAccessKeyKey: "key-a"} recordedCA := postgresapp.EndpointCA{Name: "ca-a", Key: "ca-a.crt"} @@ -340,6 +353,33 @@ func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { } } + clusterName := cnpgClusterNameForApp(appName) + liveTarget := func() []client.Object { + return []client.Object{ + &cnpgtypes.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName}}, + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName + "-1", Labels: map[string]string{cnpgClusterLabel: clusterName}}}, + } + } + // A restore that stops on the access it would use must stop before it + // touches the target: the Cluster and its volumes stay, and the + // HelmRelease that renders them is not suspended. + untouched := func(t *testing.T, o outcome) { + t.Helper() + if err := o.c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}); err != nil { + t.Errorf("target Cluster: %v, want it kept", err) + } + if err := o.c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName + "-1"}, &corev1.PersistentVolumeClaim{}); err != nil { + t.Errorf("target PVC: %v, want it kept", err) + } + hr, err := o.dyn.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, postgresAppPrefix+appName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get HelmRelease: %v", err) + } + if suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend"); suspended { + t.Errorf("the target HelmRelease was suspended") + } + } + t.Run("the recorded access is patched in", func(t *testing.T) { o := restore(t, artifactDriverMetadata(t, archiveStoreA()), nil, recordedSecrets()...) patched(t, o, recordedCreds, recordedCA) @@ -395,12 +435,27 @@ func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { o := restore(t, artifactDriverMetadata(t, a), nil, secret("creds-a", "id-a", "key-a")) patched(t, o, recordedCreds, postgresapp.EndpointCA{}) }) - t.Run("an empty recorded Secret name is no credentials", func(t *testing.T) { - a := archiveStoreA() - a.S3Credentials.SecretRef.Name = "" - o := restore(t, artifactDriverMetadata(t, a), nil, secret("ca-a", "ca-a.crt")) - patched(t, o, postgresapp.S3CredentialsSecret{}, recordedCA) - }) + // The postgres chart renders a recovery only with a credentials Secret, + // so a moved archive recorded without one is restored with the + // strategy's access rather than none. + for name, edit := range map[string]func(*strategyv1alpha1.BarmanObjectStoreTemplate){ + "recorded without credentials": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.S3Credentials = nil }, + "recorded with an empty Secret name": func(s *strategyv1alpha1.BarmanObjectStoreTemplate) { s.S3Credentials.SecretRef.Name = "" }, + } { + t.Run("a moved archive "+name+": the strategy's access is patched in", func(t *testing.T) { + a := archiveStoreA() + edit(&a) + o := restore(t, artifactDriverMetadata(t, a), nil, secret("ca-a", "ca-a.crt")) + patched(t, o, strategyCreds, strategyCA) + found := false + for _, ev := range o.events { + found = found || strings.Contains(ev, "RecordedArchiveAccessUnavailable") + } + if !found { + t.Errorf("events = %v, want one saying the recorded access was not used", o.events) + } + }) + } t.Run("recorded credentials alone are checked too", func(t *testing.T) { a := archiveStoreA() @@ -438,23 +493,25 @@ func TestReconcileCNPGRestore_PatchesTheRecordedAccess(t *testing.T) { }) t.Run("a Secret read error is returned before the patch", func(t *testing.T) { - o := restore(t, artifactDriverMetadata(t, archiveStoreA()), apierrors.NewServiceUnavailable("apiserver unavailable"), recordedSecrets()...) + o := restore(t, artifactDriverMetadata(t, archiveStoreA()), apierrors.NewServiceUnavailable("apiserver unavailable"), append(recordedSecrets(), liveTarget()...)...) if o.err == nil { t.Errorf("reconcileCNPGRestore swallowed the Secret read error") } if o.app.Spec.Bootstrap.Enabled { t.Errorf("the target app was patched although the recorded access could not be checked") } + untouched(t, o) }) t.Run("an unreadable record fails the restore before the purge", func(t *testing.T) { - o := restore(t, map[string]string{cnpgDestinationPathKey: "s3://bkt-a/tenant/pg/", cnpgArchiveAccessKey: "{"}, nil) + o := restore(t, map[string]string{cnpgDestinationPathKey: "s3://bkt-a/tenant/pg/", cnpgArchiveAccessKey: "{"}, nil, liveTarget()...) if o.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Errorf("phase = %q, want Failed", o.rj.Status.Phase) } if o.app.Spec.Bootstrap.Enabled { t.Errorf("the target app was patched for recovery from a record that could not be read") } + untouched(t, o) }) } diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index c702afa88..1167afcc5 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -1785,16 +1785,21 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore } if !apiequality.Semantic.DeepEqual(sourceCredentials, rendered.BarmanObjectStore.S3Credentials) || !apiequality.Semantic.DeepEqual(sourceEndpointCA, rendered.BarmanObjectStore.EndpointCA) { - missing, err := r.cnpgArchiveAccessMissing(ctx, target.Namespace, sourceCredentials, sourceEndpointCA) - if err != nil { - return ctrl.Result{}, err + // The postgres chart renders a recovery only with a credentials + // Secret, so recorded access without one cannot restore. + missing := "it names no credentials Secret" + if sourceCredentials != nil && sourceCredentials.SecretRef.Name != "" { + missing, err = r.cnpgArchiveAccessMissing(ctx, target.Namespace, sourceCredentials, sourceEndpointCA) + if err != nil { + return ctrl.Result{}, err + } } // The recorded access cannot work without its Secrets. The // strategy's is what a restore used before the record existed, and // credentials wider than one bucket can still read the old one. // Both settings fall back together: once part of what was - // recorded is gone, the strategy's pair is the one known to - // belong together. + // recorded is missing or unusable, the strategy's pair is the + // one known to belong together. if missing != "" { r.Recorder.Eventf(restoreJob, corev1.EventTypeWarning, "RecordedArchiveAccessUnavailable", "the access Backup %s/%s was written with is unusable (%s); reaching its archive with the strategy's instead",