// SPDX-License-Identifier: Apache-2.0 package backupcontroller import ( "context" "encoding/json" "errors" "fmt" "strings" "testing" "time" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/foundationdbapp" "github.com/cozystack/cozystack/internal/backupcontroller/foundationdbtypes" ) // --------------------------------------------------------------------------- // Validation // --------------------------------------------------------------------------- func TestValidateFoundationDBApplicationRef(t *testing.T) { apps := foundationdbapp.GroupName other := "other.example.com" cases := []struct { name string ref corev1.TypedLocalObjectReference wantErr bool }{ {"happy path with apps group", corev1.TypedLocalObjectReference{Kind: "FoundationDB", Name: "x", APIGroup: &apps}, false}, {"empty apiGroup is accepted", corev1.TypedLocalObjectReference{Kind: "FoundationDB", Name: "x"}, false}, {"foreign apiGroup rejected", corev1.TypedLocalObjectReference{Kind: "FoundationDB", Name: "x", APIGroup: &other}, true}, {"wrong kind rejected", corev1.TypedLocalObjectReference{Kind: "Postgres", Name: "x", APIGroup: &apps}, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { err := validateFoundationDBApplicationRef(tc.ref) if tc.wantErr && err == nil { t.Fatalf("expected validation error, got nil") } if !tc.wantErr && err != nil { t.Fatalf("unexpected validation error: %v", err) } }) } } // --------------------------------------------------------------------------- // Cluster-name mapping (foundationdb release.prefix) // --------------------------------------------------------------------------- func TestFoundationDBClusterNameForApp(t *testing.T) { // The cozystack foundationdb ApplicationDefinition prefixes the // release name with "foundationdb-", so an app named "fdb-src" // renders apps.foundationdb.org/FoundationDBCluster // metadata.name=foundationdb-fdb-src. if got := foundationdbClusterNameForApp("fdb-src"); got != "foundationdb-fdb-src" { t.Errorf("foundationdbClusterNameForApp: got %q want foundationdb-fdb-src", got) } } // --------------------------------------------------------------------------- // Templating // --------------------------------------------------------------------------- func TestRenderFoundationDBTemplate_TemplatingApplicationName(t *testing.T) { tmpl := strategyv1alpha1.FoundationDBTemplate{ BlobStoreConfiguration: strategyv1alpha1.FoundationDBBlobStoreTemplate{ AccountName: "key@s3.example:9000", Bucket: "{{ .Parameters.bucket }}", BackupName: "{{ .Application.metadata.name }}-fdb", URLParameters: []string{"region={{ .Parameters.region }}"}, }, CustomParameters: []string{"--blob_credentials=/var/{{ .Application.metadata.name }}/creds"}, } app := newFoundationDBApp("fdb-src", "tenant") got, err := renderFoundationDBTemplate(tmpl, app, map[string]string{"bucket": "shared-bucket", "region": "us-east-1"}) if err != nil { t.Fatalf("renderFoundationDBTemplate: %v", err) } if got.BlobStoreConfiguration.Bucket != "shared-bucket" { t.Errorf("Bucket parameter not templated: got %q", got.BlobStoreConfiguration.Bucket) } if got.BlobStoreConfiguration.BackupName != "fdb-src-fdb" { t.Errorf("BackupName not templated: got %q", got.BlobStoreConfiguration.BackupName) } if len(got.BlobStoreConfiguration.URLParameters) != 1 || got.BlobStoreConfiguration.URLParameters[0] != "region=us-east-1" { t.Errorf("URLParameters not templated: got %#v", got.BlobStoreConfiguration.URLParameters) } if len(got.CustomParameters) != 1 || got.CustomParameters[0] != "--blob_credentials=/var/fdb-src/creds" { t.Errorf("CustomParameters not templated: got %#v", got.CustomParameters) } } // --------------------------------------------------------------------------- // Backup-side ensure idempotency // --------------------------------------------------------------------------- func TestEnsureFoundationDBBackup_IdempotentByLabel(t *testing.T) { c := newFoundationDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} job := newFoundationDBBackupJob("bj-1", "tenant") if err := c.Create(context.Background(), job); err != nil { t.Fatalf("seed BackupJob: %v", err) } rendered := newRenderedFoundationDBTemplate() clusterName := foundationdbClusterNameForApp("fdb-src") first, err := r.ensureFoundationDBBackup(context.Background(), job, clusterName, "7.3.63", rendered) if err != nil { t.Fatalf("first ensureFoundationDBBackup: %v", err) } second, err := r.ensureFoundationDBBackup(context.Background(), job, clusterName, "7.3.63", rendered) if err != nil { t.Fatalf("second ensureFoundationDBBackup: %v", err) } if first.Name != second.Name { t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) } list := &foundationdbtypes.FoundationDBBackupList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list backups: %v", err) } if len(list.Items) != 1 { t.Fatalf("expected exactly one operator FoundationDBBackup CR, got %d", len(list.Items)) } got := &list.Items[0] if got.Spec.ClusterName != "foundationdb-fdb-src" { t.Errorf("ClusterName: got %q want foundationdb-fdb-src", got.Spec.ClusterName) } if got.Spec.Version != "7.3.63" { // Operator OpenAPI requires spec.version; the driver injects it // from the source FoundationDBCluster.spec.version. t.Errorf("Version: got %q want 7.3.63", got.Spec.Version) } if got.Spec.BackupState != foundationdbtypes.BackupStateRunning { t.Errorf("BackupState: got %q want %q", got.Spec.BackupState, foundationdbtypes.BackupStateRunning) } if got.Spec.BlobStoreConfiguration.BackupName != "bj-1" { // Template left BackupName empty, ensure path must default it to // the BackupJob name so each Cozystack BackupJob owns a discrete // blob-store directory. t.Errorf("BlobStoreConfiguration.BackupName: got %q want bj-1 (BackupJob name fallback)", got.Spec.BlobStoreConfiguration.BackupName) } if got.Labels[backupsv1alpha1.OwningJobNameLabel] != "bj-1" { t.Errorf("OwningJobName label missing or wrong: %v", got.Labels) } if got.Labels[foundationdbClusterLabel] != "foundationdb-fdb-src" { t.Errorf("foundationdbClusterLabel missing or wrong: %v", got.Labels) } } // TestEnsureFoundationDBBackup_StopsPriorRunning pins the FDB-specific // invariant: the operator only permits one running backup directory per // cluster, so an earlier BackupJob's CR has to flip backupState=Stopped // before the new one can start. Without this gate the operator would // reject the new FoundationDBBackup or both agents would race on the // same blob-store path. func TestEnsureFoundationDBBackup_StopsPriorRunning(t *testing.T) { clusterName := foundationdbClusterNameForApp("fdb-src") prior := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "bj-0-aaa", Labels: map[string]string{ backupsv1alpha1.OwningJobNameLabel: "bj-0", backupsv1alpha1.OwningJobNamespaceLabel: "tenant", foundationdbClusterLabel: clusterName, }, }, Spec: foundationdbtypes.FoundationDBBackupSpec{ ClusterName: clusterName, BackupState: foundationdbtypes.BackupStateRunning, }, } c := newFoundationDBStrategyTestClient(t, prior) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} job := newFoundationDBBackupJob("bj-1", "tenant") rendered := newRenderedFoundationDBTemplate() if _, err := r.ensureFoundationDBBackup(context.Background(), job, clusterName, "7.3.63", rendered); err != nil { t.Fatalf("ensureFoundationDBBackup: %v", err) } got := &foundationdbtypes.FoundationDBBackup{} if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: prior.Name}, got); err != nil { t.Fatalf("get prior CR: %v", err) } if got.Spec.BackupState != foundationdbtypes.BackupStateStopped { t.Errorf("prior FoundationDBBackup must be flipped to Stopped; got %q", got.Spec.BackupState) } } // --------------------------------------------------------------------------- // Restore-side ensure idempotency + target rewrite // --------------------------------------------------------------------------- func TestEnsureFoundationDBRestore_IdempotentByLabel(t *testing.T) { c := newFoundationDBStrategyTestClient(t) r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()} rj := newFoundationDBRestoreJob("rj-1", "tenant") if err := c.Create(context.Background(), rj); err != nil { t.Fatalf("seed RestoreJob: %v", err) } blob := foundationdbtypes.BlobStoreConfiguration{ AccountName: "key@s3.example:9000", Bucket: "bkt", BackupName: "bj-1", } first, err := r.ensureFoundationDBRestore(context.Background(), rj, "foundationdb-fdb-target", blob, nil, nil) if err != nil { t.Fatalf("first ensureFoundationDBRestore: %v", err) } second, err := r.ensureFoundationDBRestore(context.Background(), rj, "foundationdb-fdb-target", blob, nil, nil) if err != nil { t.Fatalf("second ensureFoundationDBRestore: %v", err) } if first.Name != second.Name { t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) } list := &foundationdbtypes.FoundationDBRestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list restores: %v", err) } if len(list.Items) != 1 { t.Fatalf("expected exactly one operator FoundationDBRestore CR, got %d", len(list.Items)) } got := &list.Items[0] if got.Spec.DestinationClusterName != "foundationdb-fdb-target" { t.Errorf("DestinationClusterName: got %q want foundationdb-fdb-target", got.Spec.DestinationClusterName) } if !apiequality.Semantic.DeepEqual(got.Spec.BlobStoreConfiguration, blob) { t.Errorf("BlobStoreConfiguration: got %#v want %#v", got.Spec.BlobStoreConfiguration, blob) } } // --------------------------------------------------------------------------- // resolveFoundationDBRestoreTarget (in-place vs to-copy) // --------------------------------------------------------------------------- func TestResolveFoundationDBRestoreTarget(t *testing.T) { apps := foundationdbapp.GroupName backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, }, } t.Run("in-place: missing targetApplicationRef inherits source", func(t *testing.T) { rj := &backupsv1alpha1.RestoreJob{Spec: backupsv1alpha1.RestoreJobSpec{}} r := &RestoreJobReconciler{} got := r.resolveFoundationDBRestoreTarget(rj, backup) if got.AppName != "fdb-src" { t.Errorf("in-place AppName: got %q want fdb-src", got.AppName) } if got.Kind != "FoundationDB" { t.Errorf("in-place Kind: got %q want FoundationDB", got.Kind) } }) t.Run("to-copy: targetApplicationRef wins over source", func(t *testing.T) { rj := &backupsv1alpha1.RestoreJob{ Spec: backupsv1alpha1.RestoreJobSpec{ TargetApplicationRef: &corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-target", APIGroup: &apps, }, }, } r := &RestoreJobReconciler{} got := r.resolveFoundationDBRestoreTarget(rj, backup) if got.AppName != "fdb-target" { t.Errorf("to-copy AppName: got %q want fdb-target", got.AppName) } }) } // --------------------------------------------------------------------------- // Snapshot persistence round-trip // --------------------------------------------------------------------------- func TestMarshalDecodeFoundationDBBackupSnapshot_RoundTrip(t *testing.T) { rendered := newRenderedFoundationDBTemplate() parameters := map[string]string{"region": "us-east-1"} fdbBackup := &foundationdbtypes.FoundationDBBackup{ Spec: foundationdbtypes.FoundationDBBackupSpec{ BlobStoreConfiguration: foundationdbtypes.BlobStoreConfiguration{ BackupName: "bj-1", }, }, } raw, err := marshalFoundationDBBackupSnapshot(newFoundationDBApp("fdb-src", "tenant"), rendered, parameters, fdbBackup) if err != nil { t.Fatalf("marshal: %v", err) } if raw == nil || len(raw.Raw) == 0 { t.Fatalf("expected non-empty RawExtension") } var snap foundationdbBackupSnapshot if err := json.Unmarshal(raw.Raw, &snap); err != nil { t.Fatalf("decode snapshot: %v", err) } if snap.Kind != foundationdbBackupSnapshotKind { t.Errorf("snapshot kind: got %q want %q", snap.Kind, foundationdbBackupSnapshotKind) } // Template left BackupName empty; marshal must backfill it from the // per-run FoundationDBBackup so restore-time reuse picks the right // blob-store path. if snap.Storage.BackupName != "bj-1" { t.Errorf("snapshot Storage.BackupName: got %q want bj-1 (backfilled from FoundationDBBackup)", snap.Storage.BackupName) } if got := snap.Parameters["region"]; got != "us-east-1" { t.Errorf("snapshot parameters: got %#v", snap.Parameters) } decoded, err := decodeFoundationDBBackupSnapshot(raw) if err != nil { t.Fatalf("decode: %v", err) } if decoded == nil || decoded.Kind != foundationdbBackupSnapshotKind { t.Errorf("decoded snapshot mismatch: %#v", decoded) } } // --------------------------------------------------------------------------- // resolveFoundationDBRestoreBlob: snapshot vs driverMetadata fallback // --------------------------------------------------------------------------- func TestResolveFoundationDBRestoreBlob_SnapshotWinsOverMetadata(t *testing.T) { snap := &foundationdbBackupSnapshot{ Storage: strategyv1alpha1.FoundationDBBlobStoreTemplate{ AccountName: "from-snap", Bucket: "snap-bucket", BackupName: "snap-name", }, } backup := &backupsv1alpha1.Backup{ Spec: backupsv1alpha1.BackupSpec{ DriverMetadata: map[string]string{ foundationdbAccountNameKey: "from-md", foundationdbBucketKey: "md-bucket", foundationdbBlobBackupNameKey: "md-name", }, }, } got, ok := resolveFoundationDBRestoreBlob(backup, snap) if !ok { t.Fatalf("expected ok=true with snapshot") } if got.AccountName != "from-snap" || got.Bucket != "snap-bucket" || got.BackupName != "snap-name" { t.Errorf("snapshot must win over driverMetadata; got %#v", got) } } func TestResolveFoundationDBRestoreBlob_MetadataFallback(t *testing.T) { backup := &backupsv1alpha1.Backup{ Spec: backupsv1alpha1.BackupSpec{ DriverMetadata: map[string]string{ foundationdbAccountNameKey: "from-md", foundationdbBucketKey: "md-bucket", foundationdbBlobBackupNameKey: "md-name", }, }, } got, ok := resolveFoundationDBRestoreBlob(backup, nil) if !ok { t.Fatalf("expected ok=true with driverMetadata fallback") } if got.AccountName != "from-md" || got.Bucket != "md-bucket" || got.BackupName != "md-name" { t.Errorf("driverMetadata fallback: got %#v", got) } } func TestResolveFoundationDBRestoreBlob_EmptyReturnsFalse(t *testing.T) { backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{}} _, ok := resolveFoundationDBRestoreBlob(backup, nil) if ok { t.Fatalf("empty backup must return ok=false") } } // --------------------------------------------------------------------------- // Backup readiness // --------------------------------------------------------------------------- func TestFoundationDBBackupReady(t *testing.T) { t.Run("nil backup is not ready", func(t *testing.T) { if foundationdbBackupReady(nil) { t.Fatalf("nil backup must not be ready") } }) t.Run("missing backupDetails is not ready", func(t *testing.T) { if foundationdbBackupReady(&foundationdbtypes.FoundationDBBackup{}) { t.Fatalf("backup without details must not be ready") } }) t.Run("not running is not ready", func(t *testing.T) { b := &foundationdbtypes.FoundationDBBackup{Status: foundationdbtypes.FoundationDBBackupStatus{ BackupDetails: &foundationdbtypes.BackupDetails{Running: false, SnapshotTime: 100}, }} if foundationdbBackupReady(b) { t.Fatalf("running=false must not be ready") } }) t.Run("snapshotTime=0 is not ready", func(t *testing.T) { b := &foundationdbtypes.FoundationDBBackup{Status: foundationdbtypes.FoundationDBBackupStatus{ BackupDetails: &foundationdbtypes.BackupDetails{Running: true, SnapshotTime: 0}, }} if foundationdbBackupReady(b) { t.Fatalf("snapshotTime=0 must not be ready") } }) t.Run("generations not reconciled is not ready", func(t *testing.T) { b := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{Generation: 2}, Status: foundationdbtypes.FoundationDBBackupStatus{ BackupDetails: &foundationdbtypes.BackupDetails{Running: true, SnapshotTime: 100}, Generations: &foundationdbtypes.BackupGenerationStatus{Reconciled: 1}, }, } if foundationdbBackupReady(b) { t.Fatalf("generations.reconciled < generation must not be ready") } }) t.Run("ready when running + snapshotTime > 0 + reconciled", func(t *testing.T) { b := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{Generation: 1}, Status: foundationdbtypes.FoundationDBBackupStatus{ BackupDetails: &foundationdbtypes.BackupDetails{Running: true, SnapshotTime: 100}, Generations: &foundationdbtypes.BackupGenerationStatus{Reconciled: 1}, }, } if !foundationdbBackupReady(b) { t.Fatalf("expected ready=true") } }) } // --------------------------------------------------------------------------- // Restore options + deadline // --------------------------------------------------------------------------- func TestParseFoundationDBRestoreOptions(t *testing.T) { cases := []struct { name string raw string want int64 wantErr bool }{ {"nil blob", "", 0, false}, {"missing fields", `{"foo":"bar"}`, 0, false}, {"override", `{"restoreTimeoutSeconds":7200}`, 7200, false}, {"malformed", `not-json`, 0, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var ext *runtime.RawExtension if tc.raw != "" { ext = &runtime.RawExtension{Raw: []byte(tc.raw)} } got, err := parseFoundationDBRestoreOptions(ext) if tc.wantErr { if err == nil { t.Fatalf("expected parse error, got nil") } return } if err != nil { t.Fatalf("unexpected parse error: %v", err) } if got.RestoreTimeoutSeconds != tc.want { t.Errorf("RestoreTimeoutSeconds: got %d want %d", got.RestoreTimeoutSeconds, tc.want) } }) } } func TestFoundationDBRestoreOptions_EffectiveDeadline(t *testing.T) { cases := []struct { name string opts FoundationDBRestoreOptions want time.Duration }{ {"unset", FoundationDBRestoreOptions{}, foundationdbDefaultRestoreDeadline}, {"zero", FoundationDBRestoreOptions{RestoreTimeoutSeconds: 0}, foundationdbDefaultRestoreDeadline}, {"negative", FoundationDBRestoreOptions{RestoreTimeoutSeconds: -1}, foundationdbDefaultRestoreDeadline}, {"override", FoundationDBRestoreOptions{RestoreTimeoutSeconds: 7200}, 2 * time.Hour}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := tc.opts.effectiveRestoreDeadline(); got != tc.want { t.Errorf("got %s want %s", got, tc.want) } }) } } func TestFoundationDBBackupDeadlineExceeded(t *testing.T) { if foundationdbBackupDeadlineExceeded(nil) { t.Fatalf("nil StartedAt must not trip the gate (deadline starts when StartedAt is set)") } recent := metav1.NewTime(time.Now()) if foundationdbBackupDeadlineExceeded(&recent) { t.Fatalf("recent StartedAt must not trip the gate") } old := metav1.NewTime(time.Now().Add(-2 * foundationdbDefaultBackupDeadline)) if !foundationdbBackupDeadlineExceeded(&old) { t.Fatalf("StartedAt past default deadline must trip the gate") } } // --------------------------------------------------------------------------- // URI synthesis // --------------------------------------------------------------------------- func TestFoundationDBBackupURI_PrefersOperatorReportedURL(t *testing.T) { fdb := &foundationdbtypes.FoundationDBBackup{ Spec: foundationdbtypes.FoundationDBBackupSpec{ BlobStoreConfiguration: foundationdbtypes.BlobStoreConfiguration{ AccountName: "key@s3.example:9000", Bucket: "bkt", BackupName: "bj-1", }, }, Status: foundationdbtypes.FoundationDBBackupStatus{ BackupDetails: &foundationdbtypes.BackupDetails{URL: "blobstore://bkt/bj-1?secure_connection=0"}, }, } if got, want := foundationdbBackupURI(fdb), "blobstore://bkt/bj-1?secure_connection=0"; got != want { t.Errorf("URI: got %q want %q", got, want) } } func TestFoundationDBBackupURI_FallsBackToSpec(t *testing.T) { fdb := &foundationdbtypes.FoundationDBBackup{ Spec: foundationdbtypes.FoundationDBBackupSpec{ BlobStoreConfiguration: foundationdbtypes.BlobStoreConfiguration{ AccountName: "key@s3.example:9000", Bucket: "bkt", BackupName: "bj-1", }, }, } if got, want := foundationdbBackupURI(fdb), "blobstore://bkt/bj-1"; got != want { t.Errorf("URI: got %q want %q", got, want) } } func TestFoundationDBBackupURI_EmptyIsEmpty(t *testing.T) { if got := foundationdbBackupURI(nil); got != "" { t.Errorf("nil URI: got %q want \"\"", got) } if got := foundationdbBackupURI(&foundationdbtypes.FoundationDBBackup{}); got != "" { t.Errorf("empty URI: got %q want \"\"", got) } } // TestFoundationDBBackupStateConstants_OnlyReadableValues pins #12 from // review: the driver writes spec.backupState to either "Running" (start a // backup) or "Stopped" (close a prior backup directory). The operator's // CRD enum also defines "Paused", but no code path in this driver reads // or writes it - so foundationdbtypes.BackupStatePaused was removed to // avoid an importable dead identifier that a future reader could misuse. // This test pins the surviving constants' values so a refactor that // renames them (or quietly reintroduces a dead "Paused" constant) has to // update the assertion as well. func TestFoundationDBBackupStateConstants_OnlyReadableValues(t *testing.T) { if foundationdbtypes.BackupStateRunning != "Running" { t.Errorf("BackupStateRunning: got %q want \"Running\"", foundationdbtypes.BackupStateRunning) } if foundationdbtypes.BackupStateStopped != "Stopped" { t.Errorf("BackupStateStopped: got %q want \"Stopped\"", foundationdbtypes.BackupStateStopped) } } // TestFoundationDBBackupURI_EmptyBucketIsEmpty pins #10 from review: when // bucket is unset the helper must NOT fall back to AccountName as the // bucket-slot value. AccountName looks like "@", so // the resulting URL "blobstore://@/" matches // nothing the operator actually writes and would mislead anyone trying // to locate the artefact in object storage. The helper is informational // only (load-bearing coordinates live on Backup.status.driverMetadata), // so the right answer is "". func TestFoundationDBBackupURI_EmptyBucketIsEmpty(t *testing.T) { fdb := &foundationdbtypes.FoundationDBBackup{ Spec: foundationdbtypes.FoundationDBBackupSpec{ BlobStoreConfiguration: foundationdbtypes.BlobStoreConfiguration{ AccountName: "key@s3.example:9000", BackupName: "bj-1", // Bucket intentionally empty. }, }, } if got := foundationdbBackupURI(fdb); got != "" { t.Errorf("empty bucket must produce empty URI; got %q (must not graft AccountName into the bucket slot)", got) } } // --------------------------------------------------------------------------- // Restore reconcile: target cluster transient + kind validation // --------------------------------------------------------------------------- func TestReconcileFoundationDBRestore_TargetClusterNotFoundIsTransient(t *testing.T) { apps := foundationdbapp.GroupName now := metav1.Now() cozyBackup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-cozy-bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, DriverMetadata: map[string]string{ foundationdbAccountNameKey: "key@s3.example:9000", foundationdbBucketKey: "bkt", foundationdbBlobBackupNameKey: "src-op-bk", }, }, } rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj-target-missing"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: cozyBackup.Name}, TargetApplicationRef: &corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "target", APIGroup: &apps, }, }, Status: backupsv1alpha1.RestoreJobStatus{ StartedAt: &now, }, } c := newFoundationDBStrategyTestClient(t, cozyBackup, rj) r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} res, err := r.reconcileFoundationDBRestore(context.Background(), rj, cozyBackup) if err != nil { t.Fatalf("reconcileFoundationDBRestore: %v", err) } if res.RequeueAfter == 0 { t.Errorf("missing target cluster must produce transient requeue, got %+v", res) } if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { t.Errorf("missing target cluster must NOT mark RestoreJob Failed; got phase=%q", rj.Status.Phase) } cond := apimeta.FindStatusCondition(rj.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if cond.Status != metav1.ConditionFalse || cond.Reason != "TargetFoundationDBClusterNotReady" { t.Errorf("expected Ready=False/TargetFoundationDBClusterNotReady, got %s/%s", cond.Status, cond.Reason) } } func TestReconcileFoundationDBRestore_KindCheckedBeforeBlobLookup(t *testing.T) { apps := foundationdbapp.GroupName now := metav1.Now() cozyBackup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-cozy-bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, DriverMetadata: map[string]string{ foundationdbAccountNameKey: "key@s3.example:9000", foundationdbBucketKey: "bkt", foundationdbBlobBackupNameKey: "src-op-bk", }, }, } rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj-bad-kind"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: cozyBackup.Name}, TargetApplicationRef: &corev1.TypedLocalObjectReference{ Kind: "Postgres", Name: "wrong", APIGroup: &apps, }, }, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now}, } c := newFoundationDBStrategyTestClient(t, cozyBackup, rj) r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} if _, err := r.reconcileFoundationDBRestore(context.Background(), rj, cozyBackup); err != nil { t.Fatalf("reconcileFoundationDBRestore: %v", err) } if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("expected RestoreJob to be Failed on bad TargetApplicationRef.Kind, got phase=%q", rj.Status.Phase) } cond := apimeta.FindStatusCondition(rj.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if !strings.Contains(cond.Message, "applicationRef.kind") { t.Errorf("expected Kind-rejection message, got %q", cond.Message) } } // --------------------------------------------------------------------------- // Test helpers // --------------------------------------------------------------------------- func newFoundationDBStrategyTestClient(t *testing.T, objs ...client.Object) client.Client { t.Helper() s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = foundationdbtypes.AddToScheme(s) _ = foundationdbapp.AddToScheme(s) return clientfake.NewClientBuilder(). WithScheme(s). WithObjects(objs...). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). Build() } func newFoundationDBApp(name, namespace string) *foundationdbapp.FoundationDB { return &foundationdbapp.FoundationDB{ TypeMeta: metav1.TypeMeta{ APIVersion: foundationdbapp.GroupVersion.String(), Kind: foundationdbapp.Kind, }, ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, } } func newFoundationDBBackupJob(name, namespace string) *backupsv1alpha1.BackupJob { apps := foundationdbapp.GroupName return &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, }, } } func newFoundationDBRestoreJob(name, namespace string) *backupsv1alpha1.RestoreJob { return &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: "src-backup"}, }, } } func newRenderedFoundationDBTemplate() *strategyv1alpha1.FoundationDBTemplate { return &strategyv1alpha1.FoundationDBTemplate{ BlobStoreConfiguration: strategyv1alpha1.FoundationDBBlobStoreTemplate{ AccountName: "key@s3.example:9000", Bucket: "bkt", }, CustomParameters: []string{"--blob_credentials=/var/fdb-blob-credentials/credentials"}, } } // --------------------------------------------------------------------------- // Cluster gates on the BackupJob reconcile path // --------------------------------------------------------------------------- // reconcileFoundationDBSeed assembles the minimum object graph // reconcileFoundationDB walks before reaching the FoundationDBBackup // ensure path: the strategy CR (cluster-scoped), the source application // (apps.cozystack.io/FoundationDB), and the operator-side // FoundationDBCluster the test wants to vary. Returns the seeded BackupJob // (StartedAt prefilled to skip the bootstrap branch) and the // ResolvedBackupConfig the dispatcher would have produced. func reconcileFoundationDBSeed( t *testing.T, clusterSpec foundationdbtypes.FoundationDBClusterSpec, clusterStatus foundationdbtypes.FoundationDBClusterStatus, ) (client.Client, *backupsv1alpha1.BackupJob, *ResolvedBackupConfig) { t.Helper() apps := foundationdbapp.GroupName strategyAPIGroup := strategyv1alpha1.GroupVersion.Group strategy := &strategyv1alpha1.FoundationDB{ ObjectMeta: metav1.ObjectMeta{Name: "fdb-strategy"}, Spec: strategyv1alpha1.FoundationDBSpec{Template: *newRenderedFoundationDBTemplate()}, } app := newFoundationDBApp("fdb-src", "tenant") cluster := &foundationdbtypes.FoundationDBCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: foundationdbClusterNameForApp("fdb-src"), }, Spec: clusterSpec, Status: clusterStatus, } now := metav1.NewTime(time.Now().Add(-1 * time.Minute)) job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, }, // Pre-fill StartedAt so reconcileFoundationDB skips the // stale-cache bootstrap requeue and reaches the cluster gate in // the same call. Without this, the cluster gate test would // return on the bootstrap branch and the assertion would fire // against the wrong code path. Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, } c := newFoundationDBStrategyTestClient(t, strategy, app, cluster, job) resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyAPIGroup, Kind: strategyv1alpha1.FoundationDBStrategyKind, Name: strategy.Name, }, Parameters: map[string]string{}, } // Re-fetch the BackupJob through the client so the in-test handle // carries the resourceVersion the fake client assigned at seed time; // otherwise a subsequent Status().Update from the reconcile path sees // a stale RV and returns 409. fresh := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), fresh); err != nil { t.Fatalf("seed Get BackupJob: %v", err) } return c, fresh, resolved } // TestReconcileFoundationDB_VersionEmptyIsTransient pins the gate // introduced when the operator's OpenAPI schema started requiring // FoundationDBBackup.spec.version. Before the gate the driver would // submit a CR with an empty spec.version, the apiserver would reject it, // and the BackupJob would terminally Fail until the cluster caught up. // The fix requeues with Ready=False/FoundationDBClusterVersionUnknown // instead. This test fails without the gate. func TestReconcileFoundationDB_VersionEmptyIsTransient(t *testing.T) { c, job, resolved := reconcileFoundationDBSeed( t, foundationdbtypes.FoundationDBClusterSpec{Version: ""}, foundationdbtypes.FoundationDBClusterStatus{ Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}, }, ) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} res, err := r.reconcileFoundationDB(context.Background(), job, resolved) if err != nil { t.Fatalf("reconcileFoundationDB: %v", err) } if res.RequeueAfter == 0 { t.Errorf("empty cluster.spec.version must produce transient requeue, got %+v", res) } if job.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Errorf("empty cluster.spec.version must NOT mark BackupJob Failed; got phase=%q", job.Status.Phase) } cond := apimeta.FindStatusCondition(job.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if cond.Status != metav1.ConditionFalse || cond.Reason != "FoundationDBClusterVersionUnknown" { t.Errorf("expected Ready=False/FoundationDBClusterVersionUnknown, got %s/%s", cond.Status, cond.Reason) } // The driver must NOT have materialised a FoundationDBBackup with // an empty version - that would tickle the apiserver-side OpenAPI // rejection that the gate exists to avoid. list := &foundationdbtypes.FoundationDBBackupList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list FoundationDBBackup: %v", err) } if len(list.Items) > 0 { t.Errorf("expected zero FoundationDBBackup CRs before the cluster reports a version; got %d", len(list.Items)) } } // TestReconcileFoundationDB_HealthUnavailableIsTransient pins the gate // that defers the backup until the operator reports // FoundationDBCluster.status.health.available=true. Without it, a // freshly-rendered cluster (HelmRelease applied but processes not yet // reconciled) would race into FoundationDBBackup creation against a // cluster that hasn't joined a quorum yet; the operator would either // stall or fail the start-backup transaction. The fix produces a // Ready=False/FoundationDBClusterNotAvailable requeue instead. func TestReconcileFoundationDB_HealthUnavailableIsTransient(t *testing.T) { c, job, resolved := reconcileFoundationDBSeed( t, foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"}, foundationdbtypes.FoundationDBClusterStatus{ Health: foundationdbtypes.FoundationDBClusterHealth{Available: false}, }, ) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} res, err := r.reconcileFoundationDB(context.Background(), job, resolved) if err != nil { t.Fatalf("reconcileFoundationDB: %v", err) } if res.RequeueAfter == 0 { t.Errorf("health.available=false must produce transient requeue, got %+v", res) } if job.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Errorf("health.available=false must NOT mark BackupJob Failed; got phase=%q", job.Status.Phase) } cond := apimeta.FindStatusCondition(job.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if cond.Status != metav1.ConditionFalse || cond.Reason != "FoundationDBClusterNotAvailable" { t.Errorf("expected Ready=False/FoundationDBClusterNotAvailable, got %s/%s", cond.Status, cond.Reason) } list := &foundationdbtypes.FoundationDBBackupList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list FoundationDBBackup: %v", err) } if len(list.Items) > 0 { t.Errorf("expected zero FoundationDBBackup CRs while the cluster is unavailable; got %d", len(list.Items)) } } // --------------------------------------------------------------------------- // Backup-artifact takenAt semantics // --------------------------------------------------------------------------- // TestCreateFoundationDBBackupArtifact_TakenAtIsWallClock pins the // semantics noted in the artifact builder's comment block: the FDB // operator surfaces backupDetails.snapshotTime as an FDB-internal // read-version (an integer counter, not a Unix timestamp), so the // Cozystack Backup.spec.takenAt must come from wall-clock "now" instead. // Before this regression test a no-op conditional reassigned takenAt with // the same wall-clock value, hiding the intent and inviting a future // refactor to mis-convert SnapshotTime (e.g. into time.Unix(... , 0)) and // produce a year-1970 takenAt. The check below would catch that // regression: a SnapshotTime of 43017506 interpreted as a Unix timestamp // lands in 1971. func TestCreateFoundationDBBackupArtifact_TakenAtIsWallClock(t *testing.T) { apps := foundationdbapp.GroupName strategyAPIGroup := strategyv1alpha1.GroupVersion.Group job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-wallclock"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, }, } c := newFoundationDBStrategyTestClient(t, job) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyAPIGroup, Kind: strategyv1alpha1.FoundationDBStrategyKind, Name: "fdb-strategy", }, Parameters: map[string]string{}, } // SnapshotTime is FDB read-version - a large integer that is NOT a // Unix timestamp. If a future refactor lifts SnapshotTime into // takenAt directly, the resulting time lands far away from "now". fdbBackup := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "fdb-bk"}, Status: foundationdbtypes.FoundationDBBackupStatus{ BackupDetails: &foundationdbtypes.BackupDetails{ Running: true, SnapshotTime: 43017506, }, }, } rendered := newRenderedFoundationDBTemplate() app := newFoundationDBApp("fdb-src", "tenant") before := time.Now() artifact, err := r.createFoundationDBBackupArtifact(context.Background(), job, resolved, fdbBackup, rendered, app) if err != nil { t.Fatalf("createFoundationDBBackupArtifact: %v", err) } after := time.Now() // takenAt must fall in the wall-clock window the test observed, with a // small slack on either side for clock jitter and the helper's own // metav1.Now() calls. If SnapshotTime ever bled into takenAt the value // would land decades away from this window. got := artifact.Spec.TakenAt.Time if got.Before(before.Add(-2 * time.Second)) { t.Errorf("takenAt %s is before the call window start %s; SnapshotTime must not bleed into the wall-clock value", got, before) } if got.After(after.Add(2 * time.Second)) { t.Errorf("takenAt %s is after the call window end %s", got, after) } } // --------------------------------------------------------------------------- // Precondition: missing blob-credentials Secret (review #2) // --------------------------------------------------------------------------- // reconcileFoundationDBSeedWithTemplate is reconcileFoundationDBSeed with // an explicit strategy template override. The default helper uses the // minimal template returned by newRenderedFoundationDBTemplate(), which // has no BackupDeploymentPodTemplateSpec - tests that exercise the // PodTemplateSpec-traversal preconditions need a template that // references Secrets. func reconcileFoundationDBSeedWithTemplate( t *testing.T, tmpl strategyv1alpha1.FoundationDBTemplate, clusterSpec foundationdbtypes.FoundationDBClusterSpec, clusterStatus foundationdbtypes.FoundationDBClusterStatus, extra ...client.Object, ) (client.Client, *backupsv1alpha1.BackupJob, *ResolvedBackupConfig) { t.Helper() apps := foundationdbapp.GroupName strategyAPIGroup := strategyv1alpha1.GroupVersion.Group strategy := &strategyv1alpha1.FoundationDB{ ObjectMeta: metav1.ObjectMeta{Name: "fdb-strategy"}, Spec: strategyv1alpha1.FoundationDBSpec{Template: tmpl}, } app := newFoundationDBApp("fdb-src", "tenant") cluster := &foundationdbtypes.FoundationDBCluster{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: foundationdbClusterNameForApp("fdb-src"), }, Spec: clusterSpec, Status: clusterStatus, } now := metav1.NewTime(time.Now().Add(-1 * time.Minute)) job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, }, Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, } objs := append([]client.Object{strategy, app, cluster, job}, extra...) c := newFoundationDBStrategyTestClient(t, objs...) resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyAPIGroup, Kind: strategyv1alpha1.FoundationDBStrategyKind, Name: strategy.Name, }, Parameters: map[string]string{}, } fresh := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), fresh); err != nil { t.Fatalf("seed Get BackupJob: %v", err) } return c, fresh, resolved } // templateReferencingSecret returns a minimal strategy template whose // BackupDeploymentPodTemplateSpec mounts a Secret by the given name (and // references the same Secret via envFrom to exercise both traversal // paths in podTemplateSpecMissingSecrets). func templateReferencingSecret(secretName string) strategyv1alpha1.FoundationDBTemplate { tmpl := *newRenderedFoundationDBTemplate() tmpl.BackupDeploymentPodTemplateSpec = &corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "foundationdb", EnvFrom: []corev1.EnvFromSource{{ SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, }, }}, VolumeMounts: []corev1.VolumeMount{{ Name: "blob-credentials", MountPath: "/var/fdb-blob-credentials", ReadOnly: true, }}, }}, Volumes: []corev1.Volume{{ Name: "blob-credentials", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{SecretName: secretName}, }, }}, }, } return tmpl } // TestPodTemplateSpecMissingSecrets_DedupesAndFiltersOptional pins the // behaviour of the traversal helper used by the missing-Secret // precondition: the same Secret referenced by both `volumes` and // `envFrom` should be reported once, and optional references should be // skipped. Catches a regression where someone touches the helper and // loses dedup (the reconcile loop would then Get the same Secret // twice per reconcile, harmless but wasteful) or starts reporting // optional refs (which would cause spurious "missing Secret" failures // for opt-in mounts the operator is expected to tolerate). func TestPodTemplateSpecMissingSecrets_DedupesAndFiltersOptional(t *testing.T) { tr := true tmpl := &corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: "x", EnvFrom: []corev1.EnvFromSource{ {SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{Name: "creds"}, }}, {SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{Name: "optional-extras"}, Optional: &tr, }}, }, }}, Volumes: []corev1.Volume{ {Name: "v1", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "creds"}}}, {Name: "v2", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "ca"}}}, {Name: "v3", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "opt-ca", Optional: &tr}}}, }, }, } got := podTemplateSpecMissingSecrets(tmpl) want := map[string]bool{"creds": true, "ca": true} for _, name := range got { if !want[name] { t.Errorf("unexpected Secret %q reported (optional refs must be skipped)", name) } delete(want, name) } if len(want) > 0 { t.Errorf("missing Secrets not reported: %v", want) } // And ensure no duplicates. seen := map[string]int{} for _, name := range got { seen[name]++ } for name, n := range seen { if n > 1 { t.Errorf("Secret %q reported %d times; expected exactly once", name, n) } } } // TestReconcileFoundationDB_MissingBlobCredsSecretIsTransient pins #2 from // review. A strategy template whose PodTemplateSpec mounts a per-app // blob-credentials Secret must NOT cause the BackupJob to spin until the // 45-minute deadline when the tenant forgot to pre-create the Secret; // the driver must surface a fast Ready=False/MissingBlobCredentialsSecret // requeue and not materialise a FoundationDBBackup CR (whose backup_agent // Deployment would crash-loop). func TestReconcileFoundationDB_MissingBlobCredsSecretIsTransient(t *testing.T) { secretName := "fdb-src-fdb-backup-creds" tmpl := templateReferencingSecret(secretName) c, job, resolved := reconcileFoundationDBSeedWithTemplate( t, tmpl, foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"}, foundationdbtypes.FoundationDBClusterStatus{ Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}, }, ) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} res, err := r.reconcileFoundationDB(context.Background(), job, resolved) if err != nil { t.Fatalf("reconcileFoundationDB: %v", err) } if res.RequeueAfter == 0 { t.Errorf("missing blob-creds Secret must produce transient requeue, got %+v", res) } if job.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Errorf("missing Secret must NOT mark BackupJob Failed; got phase=%q", job.Status.Phase) } cond := apimeta.FindStatusCondition(job.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if cond.Status != metav1.ConditionFalse || cond.Reason != "MissingBlobCredentialsSecret" { t.Errorf("expected Ready=False/MissingBlobCredentialsSecret, got %s/%s", cond.Status, cond.Reason) } if !strings.Contains(cond.Message, secretName) { t.Errorf("expected Ready message to name the missing Secret %q; got %q", secretName, cond.Message) } list := &foundationdbtypes.FoundationDBBackupList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list FoundationDBBackup: %v", err) } if len(list.Items) > 0 { t.Errorf("expected zero FoundationDBBackup CRs while Secret is missing; got %d", len(list.Items)) } } // TestReconcileFoundationDB_MissingBlobCredsSecretPresentProceeds is the // positive companion: when the same Secret IS present, the reconcile // must move past the precondition and materialise the FoundationDBBackup. // Without this companion the test pair could silently regress to "always // requeue on a non-empty PodTemplateSpec" and the negative test would // still pass. func TestReconcileFoundationDB_MissingBlobCredsSecretPresentProceeds(t *testing.T) { secretName := "fdb-src-fdb-backup-creds" tmpl := templateReferencingSecret(secretName) creds := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: secretName}, Type: corev1.SecretTypeOpaque, } c, job, resolved := reconcileFoundationDBSeedWithTemplate( t, tmpl, foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"}, foundationdbtypes.FoundationDBClusterStatus{ Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}, }, creds, ) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} if _, err := r.reconcileFoundationDB(context.Background(), job, resolved); err != nil { t.Fatalf("reconcileFoundationDB: %v", err) } list := &foundationdbtypes.FoundationDBBackupList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list FoundationDBBackup: %v", err) } if len(list.Items) != 1 { t.Fatalf("expected exactly one FoundationDBBackup once Secret is present; got %d", len(list.Items)) } } // --------------------------------------------------------------------------- // Precondition: conflicting in-chart backup (review #3) // --------------------------------------------------------------------------- // TestReconcileFoundationDB_ConflictingInChartBackupIsFatal pins #3 from // review. A chart-rendered FoundationDBBackup (no driver labels, Running, // targeting the same cluster) must produce a terminal BackupJob failure // with the conflict-specific reason. Silently stopping the chart's CR // would split the in-chart user's backup stream and corrupt both flows. // stopOtherFoundationDBBackups labels-on-list cannot find the chart CR, // so this gate is the only thing standing between the two. func TestReconcileFoundationDB_ConflictingInChartBackupIsFatal(t *testing.T) { chartBackup := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "foundationdb-fdb-src-backup", // chart-rendered name // Critically: no driver labels. }, Spec: foundationdbtypes.FoundationDBBackupSpec{ ClusterName: foundationdbClusterNameForApp("fdb-src"), Version: "7.3.63", BackupState: foundationdbtypes.BackupStateRunning, }, } c, job, resolved := reconcileFoundationDBSeedWithTemplate( t, *newRenderedFoundationDBTemplate(), foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"}, foundationdbtypes.FoundationDBClusterStatus{ Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}, }, chartBackup, ) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} if _, err := r.reconcileFoundationDB(context.Background(), job, resolved); err != nil { t.Fatalf("reconcileFoundationDB: %v", err) } if job.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("expected BackupJob to be Failed on in-chart conflict, got phase=%q", job.Status.Phase) } if !strings.Contains(job.Status.Message, "conflicting in-chart FoundationDBBackup") { t.Errorf("expected message to mention 'conflicting in-chart FoundationDBBackup'; got %q", job.Status.Message) } if !strings.Contains(job.Status.Message, chartBackup.Name) { t.Errorf("expected message to name the conflicting CR %q; got %q", chartBackup.Name, job.Status.Message) } // The chart-rendered CR must remain untouched - the diagnostic is // loud precisely so the user can review it before deleting. got := &foundationdbtypes.FoundationDBBackup{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(chartBackup), got); err != nil { t.Fatalf("get chart backup post-reconcile: %v", err) } if got.Spec.BackupState != foundationdbtypes.BackupStateRunning { t.Errorf("driver must not modify the chart-rendered CR's spec.backupState; got %q", got.Spec.BackupState) } } // TestFindConflictingInChartBackup_IgnoresDriverManagedCRs pins the // label-key gate inside findConflictingInChartBackup: a Running // FoundationDBBackup that DOES carry foundationdbClusterLabel is a // driver-managed CR (e.g. a prior BackupJob's leftover), not a // chart-rendered conflict, and must be passed through to // stopOtherFoundationDBBackups instead of failing the new BackupJob. func TestFindConflictingInChartBackup_IgnoresDriverManagedCRs(t *testing.T) { clusterName := foundationdbClusterNameForApp("fdb-src") driverBackup := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "bj-prev-aaa", Labels: map[string]string{ backupsv1alpha1.OwningJobNameLabel: "bj-prev", backupsv1alpha1.OwningJobNamespaceLabel: "tenant", foundationdbClusterLabel: clusterName, }, }, Spec: foundationdbtypes.FoundationDBBackupSpec{ ClusterName: clusterName, BackupState: foundationdbtypes.BackupStateRunning, }, } c := newFoundationDBStrategyTestClient(t, driverBackup) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} name, err := r.findConflictingInChartBackup(context.Background(), "tenant", clusterName) if err != nil { t.Fatalf("findConflictingInChartBackup: %v", err) } if name != "" { t.Errorf("driver-managed Running CR must NOT be reported as conflicting; got %q", name) } } // --------------------------------------------------------------------------- // stopOtherFoundationDBBackups: retry on conflict (review #6) // --------------------------------------------------------------------------- // TestStopOtherFoundationDBBackups_RetriesOnConflict pins #6 from review. // The operator reconciles spec.backupState concurrently with our patch; // a stale resourceVersion produces a 409 Conflict that previously failed // the entire BackupJob with "stop prior FoundationDBBackup ...". The fix // is to refetch + retry on 409. This test injects a single 409 on the // first Patch via an interceptor, then lets the second patch through; // the prior CR must end up Stopped and no error must propagate. func TestStopOtherFoundationDBBackups_RetriesOnConflict(t *testing.T) { clusterName := foundationdbClusterNameForApp("fdb-src") prior := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "bj-prev-bbb", Labels: map[string]string{ backupsv1alpha1.OwningJobNameLabel: "bj-prev", backupsv1alpha1.OwningJobNamespaceLabel: "tenant", foundationdbClusterLabel: clusterName, }, }, Spec: foundationdbtypes.FoundationDBBackupSpec{ ClusterName: clusterName, BackupState: foundationdbtypes.BackupStateRunning, }, } job := newFoundationDBBackupJob("bj-new", "tenant") s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = foundationdbtypes.AddToScheme(s) _ = foundationdbapp.AddToScheme(s) patchAttempts := 0 fdbBackupGR := schema.GroupResource{ Group: foundationdbtypes.GroupName, Resource: "foundationdbbackups", } c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(prior, job). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). WithInterceptorFuncs(interceptor.Funcs{ Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*foundationdbtypes.FoundationDBBackup); ok { patchAttempts++ if patchAttempts == 1 { return apierrors.NewConflict(fdbBackupGR, obj.GetName(), fmt.Errorf("simulated stale resourceVersion")) } } return cl.Patch(ctx, obj, patch, opts...) }, }). Build() r := &BackupJobReconciler{Client: c, Scheme: s} if err := r.stopOtherFoundationDBBackups(context.Background(), job, clusterName); err != nil { t.Fatalf("stopOtherFoundationDBBackups returned error despite retry budget; the 409 should have been absorbed: %v", err) } if patchAttempts < 2 { t.Errorf("expected at least 2 Patch attempts (one Conflict + one success), got %d", patchAttempts) } got := &foundationdbtypes.FoundationDBBackup{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(prior), got); err != nil { t.Fatalf("get prior post-stop: %v", err) } if got.Spec.BackupState != foundationdbtypes.BackupStateStopped { t.Errorf("prior FoundationDBBackup must be Stopped after retry; got %q", got.Spec.BackupState) } } // --------------------------------------------------------------------------- // Terminal-failure cleanup (review #7) // --------------------------------------------------------------------------- // TestReconcileFoundationDB_DeadlineExpiry_StopsFoundationDBBackup pins // #7 from review. When the BackupJob hits the 45-minute deadline before // the FoundationDBBackup lands a snapshot, the driver must Stop the // operator-side CR before marking the BackupJob Failed. Otherwise the // backup_agent Deployment continues retrying against a broken S3 // endpoint forever (one Deployment leaked per failed BackupJob) and the // FDB operator keeps reconciling against an undeleted CR. func TestReconcileFoundationDB_DeadlineExpiry_StopsFoundationDBBackup(t *testing.T) { apps := foundationdbapp.GroupName strategyAPIGroup := strategyv1alpha1.GroupVersion.Group clusterName := foundationdbClusterNameForApp("fdb-src") pastStart := metav1.NewTime(time.Now().Add(-2 * foundationdbDefaultBackupDeadline)) strategy := &strategyv1alpha1.FoundationDB{ ObjectMeta: metav1.ObjectMeta{Name: "fdb-strategy"}, Spec: strategyv1alpha1.FoundationDBSpec{Template: *newRenderedFoundationDBTemplate()}, } app := newFoundationDBApp("fdb-src", "tenant") cluster := &foundationdbtypes.FoundationDBCluster{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: clusterName}, Spec: foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"}, Status: foundationdbtypes.FoundationDBClusterStatus{ Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}, }, } job := &backupsv1alpha1.BackupJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, }, // StartedAt 2x deadline ago - foundationdbBackupDeadlineExceeded // will return true on the first reconcile that reaches the gate. Status: backupsv1alpha1.BackupJobStatus{ StartedAt: &pastStart, Phase: backupsv1alpha1.BackupJobPhaseRunning, }, } // Pre-seeded prior FoundationDBBackup with the BackupJob's labels and // no backupDetails (so foundationdbBackupReady returns false) - this // is the CR the driver must Stop on deadline expiry. stuck := &foundationdbtypes.FoundationDBBackup{ ObjectMeta: metav1.ObjectMeta{ Namespace: "tenant", Name: "bj-stuck", Labels: map[string]string{ backupsv1alpha1.OwningJobNameLabel: job.Name, backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace, foundationdbClusterLabel: clusterName, }, }, Spec: foundationdbtypes.FoundationDBBackupSpec{ ClusterName: clusterName, Version: "7.3.63", BackupState: foundationdbtypes.BackupStateRunning, }, } c := newFoundationDBStrategyTestClient(t, strategy, app, cluster, job, stuck) fresh := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), fresh); err != nil { t.Fatalf("seed Get BackupJob: %v", err) } resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{ APIGroup: &strategyAPIGroup, Kind: strategyv1alpha1.FoundationDBStrategyKind, Name: strategy.Name, }, Parameters: map[string]string{}, } r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} if _, err := r.reconcileFoundationDB(context.Background(), fresh, resolved); err != nil { t.Fatalf("reconcileFoundationDB: %v", err) } if fresh.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { t.Fatalf("expected BackupJob to be Failed on deadline expiry; got phase=%q", fresh.Status.Phase) } got := &foundationdbtypes.FoundationDBBackup{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(stuck), got); err != nil { t.Fatalf("get stuck CR post-reconcile: %v", err) } if got.Spec.BackupState != foundationdbtypes.BackupStateStopped { t.Errorf("FoundationDBBackup must be Stopped before terminal failure; got %q (backup_agent Deployment would otherwise leak past the BackupJob's lifetime)", got.Spec.BackupState) } } // --------------------------------------------------------------------------- // Malformed restoreJob.spec.options is terminal (review #11) // --------------------------------------------------------------------------- // TestReconcileFoundationDBRestore_MalformedOptionsIsTerminal pins #11 // from review. spec.options is a tenant-supplied JSON blob; when it is // malformed (e.g. user wrote `restoreTimeoutSeconds: "5h"` as a string // instead of an int64), the driver previously silently fell back to the // 30-minute default deadline and the misconfiguration would persist // until a restore exceeded that default. The fix marks the RestoreJob // Failed terminally so the tenant gets an actionable signal on the // first reconcile. This test seeds malformed options and asserts the // terminal failure plus that no FoundationDBRestore CR was created. func TestReconcileFoundationDBRestore_MalformedOptionsIsTerminal(t *testing.T) { apps := foundationdbapp.GroupName now := metav1.Now() cozyBackup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-cozy-bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, DriverMetadata: map[string]string{ foundationdbAccountNameKey: "key@s3.example:9000", foundationdbBucketKey: "bkt", foundationdbBlobBackupNameKey: "src-op-bk", }, }, } // Valid JSON, invalid against the FoundationDBRestoreOptions schema: // restoreTimeoutSeconds is declared as int64 but a tenant has typed it // as a string. parseFoundationDBRestoreOptions surfaces this as an // unmarshal error. (Using raw "not-json" would fail RawExtension's // MarshalJSON during fake-client seeding, before the code under test // runs - choose a payload that round-trips through the apiserver and // is rejected only inside parseFoundationDBRestoreOptions.) rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj-malformed"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: cozyBackup.Name}, Options: &runtime.RawExtension{Raw: []byte(`{"restoreTimeoutSeconds": "five-hours"}`)}, }, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now}, } c := newFoundationDBStrategyTestClient(t, cozyBackup, rj) r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} if _, err := r.reconcileFoundationDBRestore(context.Background(), rj, cozyBackup); err != nil { t.Fatalf("reconcileFoundationDBRestore: %v", err) } if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("malformed spec.options must mark RestoreJob Failed; got phase=%q", rj.Status.Phase) } cond := apimeta.FindStatusCondition(rj.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if !strings.Contains(cond.Message, "malformed") { t.Errorf("expected Ready message to flag malformed options; got %q", cond.Message) } list := &foundationdbtypes.FoundationDBRestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list FoundationDBRestore: %v", err) } if len(list.Items) > 0 { t.Errorf("expected zero FoundationDBRestore CRs on malformed options; got %d", len(list.Items)) } } // --------------------------------------------------------------------------- // Snapshot APIVersion/Kind contract (review #1 from lllamnyp) // --------------------------------------------------------------------------- // TestDecodeFoundationDBBackupSnapshot_RejectsMismatchedAPIVersion pins // the contract documented next to foundationdbBackupSnapshotKind: a // future v2 snapshot with the same Kind but a different APIVersion must // be rejected with errSnapshotUnrecognised, not silently parsed as v1. // Before this fix the decoder ignored APIVersion entirely; a v2 // snapshot with extra fields would have been mis-interpreted as v1. func TestDecodeFoundationDBBackupSnapshot_RejectsMismatchedAPIVersion(t *testing.T) { raw, err := json.Marshal(foundationdbBackupSnapshot{ Kind: foundationdbBackupSnapshotKind, APIVersion: "future.cozystack.io/v2", }) if err != nil { t.Fatalf("marshal: %v", err) } got, err := decodeFoundationDBBackupSnapshot(&runtime.RawExtension{Raw: raw}) if err == nil { t.Fatalf("expected decode error on mismatched APIVersion, got nil") } if !errors.Is(err, errSnapshotUnrecognised) { t.Errorf("expected error to wrap errSnapshotUnrecognised; got %v", err) } if got != nil { t.Errorf("expected nil snapshot on mismatch; got %+v", got) } } // TestDecodeFoundationDBBackupSnapshot_RejectsMismatchedKind pins the // same contract for the Kind field. Previously the decoder returned // (nil, nil) on Kind mismatch, indistinguishable from "no snapshot // present", which let a foreign payload silently fall through to the // driverMetadata fallback at the call site. func TestDecodeFoundationDBBackupSnapshot_RejectsMismatchedKind(t *testing.T) { raw, err := json.Marshal(foundationdbBackupSnapshot{ Kind: "SomeOtherSnapshot", APIVersion: foundationdbBackupSnapshotAPIVersion, }) if err != nil { t.Fatalf("marshal: %v", err) } got, err := decodeFoundationDBBackupSnapshot(&runtime.RawExtension{Raw: raw}) if err == nil { t.Fatalf("expected decode error on mismatched Kind, got nil") } if !errors.Is(err, errSnapshotUnrecognised) { t.Errorf("expected error to wrap errSnapshotUnrecognised; got %v", err) } if got != nil { t.Errorf("expected nil snapshot on mismatch; got %+v", got) } } // TestDecodeFoundationDBBackupSnapshot_NilRawReturnsNilNil pins the // "no snapshot present" path so the call site can keep distinguishing // it from the unrecognised case via err == nil + snap == nil. func TestDecodeFoundationDBBackupSnapshot_NilRawReturnsNilNil(t *testing.T) { got, err := decodeFoundationDBBackupSnapshot(nil) if err != nil || got != nil { t.Errorf("nil raw: got (%+v, %v) want (nil, nil)", got, err) } got, err = decodeFoundationDBBackupSnapshot(&runtime.RawExtension{}) if err != nil || got != nil { t.Errorf("empty raw: got (%+v, %v) want (nil, nil)", got, err) } } // TestReconcileFoundationDBRestore_SnapshotAPIVersionMismatchIsTerminal // pins the call-site half of the contract: a Backup carrying a // snapshot with a foreign APIVersion must terminate the RestoreJob // instead of silently falling back to driverMetadata. Without this // gate a v1 reader would proceed against a v2 snapshot's stale // interpretation, corrupting the restore (any v2 fields the v1 reader // doesn't carry - e.g. new CustomParameters - would be silently // dropped). func TestReconcileFoundationDBRestore_SnapshotAPIVersionMismatchIsTerminal(t *testing.T) { apps := foundationdbapp.GroupName now := metav1.Now() rawSnap, err := json.Marshal(foundationdbBackupSnapshot{ Kind: foundationdbBackupSnapshotKind, APIVersion: "future.cozystack.io/v2", }) if err != nil { t.Fatalf("marshal snap: %v", err) } cozyBackup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-cozy-bk"}, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: corev1.TypedLocalObjectReference{ Kind: "FoundationDB", Name: "fdb-src", APIGroup: &apps, }, // driverMetadata IS present - confirms the call site does // not fall back to it when the snapshot is unrecognised. DriverMetadata: map[string]string{ foundationdbAccountNameKey: "key@s3.example:9000", foundationdbBucketKey: "bkt", foundationdbBlobBackupNameKey: "src-op-bk", }, }, Status: backupsv1alpha1.BackupStatus{ UnderlyingResources: &runtime.RawExtension{Raw: rawSnap}, }, } rj := &backupsv1alpha1.RestoreJob{ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj-version-mismatch"}, Spec: backupsv1alpha1.RestoreJobSpec{ BackupRef: corev1.LocalObjectReference{Name: cozyBackup.Name}, }, Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now}, } c := newFoundationDBStrategyTestClient(t, cozyBackup, rj) r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} if _, err := r.reconcileFoundationDBRestore(context.Background(), rj, cozyBackup); err != nil { t.Fatalf("reconcileFoundationDBRestore: %v", err) } if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { t.Fatalf("unrecognised snapshot apiVersion must mark RestoreJob Failed; got phase=%q", rj.Status.Phase) } cond := apimeta.FindStatusCondition(rj.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition, got none") } if !strings.Contains(cond.Message, "unrecognised") && !strings.Contains(cond.Message, "apiVersion") { t.Errorf("expected Ready message to flag the apiVersion/kind mismatch; got %q", cond.Message) } list := &foundationdbtypes.FoundationDBRestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { t.Fatalf("list FoundationDBRestore: %v", err) } if len(list.Items) > 0 { t.Errorf("expected zero FoundationDBRestore CRs on snapshot mismatch; got %d", len(list.Items)) } } // --------------------------------------------------------------------------- // No-OwnerReference contract (review #4 from lllamnyp) // --------------------------------------------------------------------------- // TestEnsureFoundationDBBackup_DoesNotSetOwnerReference pins the // deliberate omission of OwnerReferences from the operator-side // FoundationDBBackup CR. The driver labels the CR with the BackupJob's // OwningJob{Name,Namespace} for idempotent ensure-by-label semantics // across BackupJob recreates with the same name (`kubectl delete && // kubectl apply` retries find the prior CR and reuse it instead of // leaking a duplicate backup_agent Deployment). Adding an // OwnerReference back to the BackupJob would make Kubernetes GC reap // the operator CR the moment the parent BackupJob is deleted, // defeating the reuse path. The framework's label-based cleanup // (cleanup.sh, stopFoundationDBBackupForJob) handles teardown // explicitly. // // If a future change adds OwnerReferences "for safety", this test // fails immediately so the regression is caught before users hit it. func TestEnsureFoundationDBBackup_DoesNotSetOwnerReference(t *testing.T) { c := newFoundationDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} job := newFoundationDBBackupJob("bj-no-owner", "tenant") if err := c.Create(context.Background(), job); err != nil { t.Fatalf("seed BackupJob: %v", err) } clusterName := foundationdbClusterNameForApp("fdb-src") rendered := newRenderedFoundationDBTemplate() got, err := r.ensureFoundationDBBackup(context.Background(), job, clusterName, "7.3.63", rendered) if err != nil { t.Fatalf("ensureFoundationDBBackup: %v", err) } if got == nil { t.Fatalf("expected a FoundationDBBackup, got nil") } if len(got.OwnerReferences) != 0 { t.Errorf("FoundationDBBackup must NOT carry OwnerReferences (would defeat the idempotent ensure-by-label reuse across BackupJob recreates); got %#v", got.OwnerReferences) } // Also confirm the labels that DO power the reuse path are present - // the no-OwnerReference contract only makes sense alongside them. if got.Labels[backupsv1alpha1.OwningJobNameLabel] != job.Name { t.Errorf("OwningJobName label missing or wrong: %v", got.Labels) } if got.Labels[backupsv1alpha1.OwningJobNamespaceLabel] != job.Namespace { t.Errorf("OwningJobNamespace label missing or wrong: %v", got.Labels) } } // --------------------------------------------------------------------------- // requeueWithReason retry on Status Conflict (review #5 from lllamnyp) // --------------------------------------------------------------------------- // TestRequeueWithReason_RetriesOnStatusConflict pins #5 from review. // requeueWithReason is hot on the cold-start reconcile path (six call // sites, five of which fire before the BackupJob's first successful // Status update), where the informer cache is most likely to be // stale. Before this fix a 409 Conflict on the first Status().Update // bubbled up to controller-runtime, which rescheduled the reconcile - // on a heavily-contended namespace the next reschedule could observe // the same stale RV and busy-spin until the informer caught up. // // The fix refetches the BackupJob on 409 and retries the Update with // the fresh resourceVersion (mirroring the patch-retry budget used by // stopOtherFoundationDBBackups). This test injects a single 409 on // the first Status().Update via interceptor.Funcs, then lets the // second through; the condition must land without returning an error. func TestRequeueWithReason_RetriesOnStatusConflict(t *testing.T) { job := newFoundationDBBackupJob("bj-conflict", "tenant") s := runtime.NewScheme() _ = scheme.AddToScheme(s) _ = backupsv1alpha1.AddToScheme(s) _ = strategyv1alpha1.AddToScheme(s) _ = foundationdbtypes.AddToScheme(s) _ = foundationdbapp.AddToScheme(s) patchAttempts := 0 gr := schema.GroupResource{Group: backupsv1alpha1.GroupVersion.Group, Resource: "backupjobs"} c := clientfake.NewClientBuilder(). WithScheme(s). WithObjects(job). WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). WithInterceptorFuncs(interceptor.Funcs{ SubResourceUpdate: func(ctx context.Context, cl client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { if _, ok := obj.(*backupsv1alpha1.BackupJob); ok && subResourceName == "status" { patchAttempts++ if patchAttempts == 1 { return apierrors.NewConflict(gr, obj.GetName(), fmt.Errorf("simulated stale resourceVersion")) } } return cl.Status().Update(ctx, obj, opts...) }, }). Build() r := &BackupJobReconciler{Client: c, Scheme: s} fresh := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), fresh); err != nil { t.Fatalf("seed Get: %v", err) } res, err := r.requeueWithReason(context.Background(), fresh, "TestReason", "test message") if err != nil { t.Fatalf("requeueWithReason returned error despite retry budget; the 409 should have been absorbed: %v", err) } if res.RequeueAfter == 0 { t.Errorf("requeueWithReason must still requeue after a successful retry; got %+v", res) } if patchAttempts < 2 { t.Errorf("expected at least 2 Status Update attempts (one Conflict + one success), got %d", patchAttempts) } got := &backupsv1alpha1.BackupJob{} if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), got); err != nil { t.Fatalf("post-reconcile Get: %v", err) } cond := apimeta.FindStatusCondition(got.Status.Conditions, "Ready") if cond == nil { t.Fatalf("expected Ready condition to land after retry; got none") } if cond.Reason != "TestReason" { t.Errorf("Ready reason: got %q want %q", cond.Reason, "TestReason") } }