mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-03 18:47:04 +09:00
Adds a MongoDB (Percona Server for MongoDB / psmdb) backup-strategy driver so `apps.cozystack.io/MongoDB` reaches feature parity with the CNPG, MariaDB and ClickHouse native drivers — logical backups, restore-to-differently-named instances, and point-in-time recovery through the unified BackupJob/RestoreJob/BackupClass/Plan interface (closes #3545). The driver orchestrates the operator's PerconaServerMongoDBBackup / PerconaServerMongoDBRestore CRDs: - BackupJob: creates a one-shot PerconaServerMongoDBBackup, polls status.state, and persists the S3 destination + storage descriptor (by-reference, no credentials) onto the Cozystack Backup artifact. - RestoreJob: drives PerconaServerMongoDBRestore via backupSource (read back from the source backup status, snapshot fallback) so the same artifact restores in-place or into a differently-named instance; PITR via spec.options.pitr (date/latest). Because psmdb resolves the backup storage by name against the cluster's spec.backup.storages, the driver names the chart's storage (s3-storage) via the cozy-default-mongodb Strategy rather than injecting it, and surfaces a clear Ready=False precondition when a target has backups disabled. Wiring: - api/backups/strategy/v1alpha1: MongoDB strategy type + generated CRD/deepcopy. - internal/backupcontroller: psmdbstrategy_controller + psmdbtypes/mongodbapp partial-schema mirrors; dispatch, supported-kinds and scheme registration. - backupstrategy-controller: cozy-default MongoDB route, cozy-default-mongodb Strategy CR, and psmdb.percona.com RBAC. - apps/mongodb: backup.insecureSkipTLSVerify knob (psmdb s3 storage has no CA-bundle field) so backups can target Cozystack's self-signed seaweedfs. Tests + e2e: - unit tests for the driver (validation, ensure-idempotency, precondition, backupSource resolution, snapshot round-trip, PITR option parsing). - examples/backups/mongodb + a mongodb-2-backup-roundtrip Chainsaw suite that drives run-all.sh end-to-end (sentinel document round-trips through S3 into a restored copy while the source stays untouched), per docs/agents/e2e-testing.md. - docs/operations/backup-classes.md updated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Andrey Kolkov <androndo@gmail.com>
531 lines
22 KiB
Go
531 lines
22 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
)
|
|
|
|
// TestSupportedBackupStrategyKindsMatchesDispatch pins every Kind that the
|
|
// BackupJob reconciler's dispatch switch handles. Before the fix the
|
|
// unsupported-strategy log payload was hand-maintained inline next to the
|
|
// switch and silently dropped AltinityStrategyKind when MariaDB was added,
|
|
// which misled operators staring at the message and trying to figure out
|
|
// which strategies the controller actually supports. The single
|
|
// supportedBackupStrategyKinds() source of truth keeps log and dispatch in
|
|
// lockstep; this test fails the moment a new strategy is added to either
|
|
// half but not the other.
|
|
func TestSupportedBackupStrategyKindsMatchesDispatch(t *testing.T) {
|
|
got := append([]string(nil), supportedBackupStrategyKinds()...)
|
|
want := []string{
|
|
strategyv1alpha1.JobStrategyKind,
|
|
strategyv1alpha1.VeleroStrategyKind,
|
|
strategyv1alpha1.CNPGStrategyKind,
|
|
strategyv1alpha1.AltinityStrategyKind,
|
|
strategyv1alpha1.MariaDBStrategyKind,
|
|
strategyv1alpha1.MongoDBStrategyKind,
|
|
strategyv1alpha1.FoundationDBStrategyKind,
|
|
strategyv1alpha1.EtcdStrategyKind,
|
|
}
|
|
sort.Strings(got)
|
|
sort.Strings(want)
|
|
if len(got) != len(want) {
|
|
t.Fatalf("supportedBackupStrategyKinds: got %d kinds %v, want %d %v", len(got), got, len(want), want)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Errorf("supportedBackupStrategyKinds[%d]: got %q, want %q", i, got[i], want[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestMarkBackupJobFailed_DoesNotAppendDuplicateReady locks in the fix:
|
|
// markBackupJobFailed used to `append` to Status.Conditions, which violates
|
|
// the +listType=map +listMapKey=type contract on the field. Two terminal
|
|
// failures (or a terminal failure on top of a transient Ready=False
|
|
// condition that a previous reconcile already wrote) used to leave two
|
|
// entries with Type="Ready" - any consumer doing meta.FindStatusCondition
|
|
// reads whichever copy comes first in the slice, not necessarily the
|
|
// latest. SetStatusCondition keeps the list a proper map.
|
|
func TestMarkBackupJobFailed_DoesNotAppendDuplicateReady(t *testing.T) {
|
|
t.Run("two failures keep exactly one Ready condition", func(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "bj"},
|
|
}
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c}
|
|
|
|
if _, err := r.markBackupJobFailed(context.Background(), bj, "first failure"); err != nil {
|
|
t.Fatalf("first markBackupJobFailed: %v", err)
|
|
}
|
|
if _, err := r.markBackupJobFailed(context.Background(), bj, "second failure"); err != nil {
|
|
t.Fatalf("second markBackupJobFailed: %v", err)
|
|
}
|
|
|
|
readyCount := 0
|
|
for _, cond := range bj.Status.Conditions {
|
|
if cond.Type == "Ready" {
|
|
readyCount++
|
|
}
|
|
}
|
|
if readyCount != 1 {
|
|
t.Fatalf("expected exactly 1 Ready condition, got %d (full slice: %+v)", readyCount, bj.Status.Conditions)
|
|
}
|
|
ready := meta.FindStatusCondition(bj.Status.Conditions, "Ready")
|
|
if ready == nil || ready.Message != "second failure" {
|
|
t.Errorf("expected latest failure message preserved, got %+v", ready)
|
|
}
|
|
})
|
|
|
|
t.Run("transient Ready=False condition is replaced in place", func(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "bj"},
|
|
Status: backupsv1alpha1.BackupJobStatus{
|
|
Conditions: []metav1.Condition{
|
|
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "WaitingForBackup", Message: "stale"},
|
|
},
|
|
},
|
|
}
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c}
|
|
|
|
if _, err := r.markBackupJobFailed(context.Background(), bj, "backup deadline exceeded"); err != nil {
|
|
t.Fatalf("markBackupJobFailed: %v", err)
|
|
}
|
|
if got := len(bj.Status.Conditions); got != 1 {
|
|
t.Fatalf("expected 1 condition, got %d (full slice: %+v)", got, bj.Status.Conditions)
|
|
}
|
|
if bj.Status.Conditions[0].Reason != "BackupFailed" {
|
|
t.Errorf("expected Reason=BackupFailed, got %q", bj.Status.Conditions[0].Reason)
|
|
}
|
|
if bj.Status.Conditions[0].Message != "backup deadline exceeded" {
|
|
t.Errorf("expected new message, got %q", bj.Status.Conditions[0].Message)
|
|
}
|
|
})
|
|
}
|
|
|
|
func newBackupJobTestClient(t *testing.T, objs ...client.Object) client.Client {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(s)
|
|
_ = backupsv1alpha1.AddToScheme(s)
|
|
return clientfake.NewClientBuilder().
|
|
WithScheme(s).
|
|
WithObjects(objs...).
|
|
WithStatusSubresource(&backupsv1alpha1.BackupJob{}).
|
|
Build()
|
|
}
|
|
|
|
// TestReconcile_SkipsProjection_TerminalPhase covers review finding #2:
|
|
// a Succeeded/Failed BackupJob must not keep projecting Secrets on every
|
|
// requeue. Before the fix the controller would Get the source Secret and
|
|
// Get/Update the target on each reconcile of a terminal job.
|
|
func TestReconcile_SkipsProjection_TerminalPhase(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-acme", Name: "bj"},
|
|
Status: backupsv1alpha1.BackupJobStatus{Phase: backupsv1alpha1.BackupJobPhaseSucceeded},
|
|
}
|
|
c := newBackupJobTestClient(t, bj, flatSourceSecret())
|
|
r := &BackupJobReconciler{Client: c, CredentialsConfig: defaultCfg()}
|
|
res, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 || res.Requeue {
|
|
t.Fatalf("terminal BackupJob must not requeue, got %+v", res)
|
|
}
|
|
// Projection must NOT have created the target Secret.
|
|
got := &corev1.Secret{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err == nil {
|
|
t.Fatalf("terminal BackupJob triggered projection; target Secret should be absent")
|
|
}
|
|
}
|
|
|
|
// TestReconcile_SkipsProjection_ForeignAPIGroup covers review finding #2
|
|
// + #7: a BackupJob whose BackupClass resolves to a strategy outside
|
|
// strategy.backups.cozystack.io must not project cozy-backups-creds (so
|
|
// third-party drivers and clusters with a manually-managed
|
|
// cozy-backups-creds Secret are unaffected). Before the fix projection
|
|
// ran before APIGroup filtering and could terminally fail unrelated
|
|
// BackupJobs via the ownership guard.
|
|
func TestReconcile_SkipsProjection_ForeignAPIGroup(t *testing.T) {
|
|
foreignGroup := "third-party.example.com"
|
|
bc := &backupsv1alpha1.BackupClass{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "foreign-class"},
|
|
Spec: backupsv1alpha1.BackupClassSpec{
|
|
Strategies: []backupsv1alpha1.BackupClassStrategy{
|
|
{
|
|
Application: backupsv1alpha1.ApplicationSelector{
|
|
APIGroup: stringPtr("apps.cozystack.io"),
|
|
Kind: "Postgres",
|
|
},
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &foreignGroup,
|
|
Kind: "CustomDriver",
|
|
Name: "custom",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-acme", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
BackupClassName: "foreign-class",
|
|
ApplicationRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: stringPtr("apps.cozystack.io"),
|
|
Kind: "Postgres",
|
|
Name: "pg",
|
|
},
|
|
},
|
|
}
|
|
// Plant an unowned cozy-backups-creds: the guard would terminally
|
|
// fail the BackupJob if projection ran. With the fix in place,
|
|
// projection is gated on the platform APIGroup and the unowned
|
|
// Secret is left alone.
|
|
unowned := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-acme", Name: "cozy-backups-creds"},
|
|
Data: map[string][]byte{"user-data": []byte("x")},
|
|
}
|
|
c := newBackupJobTestClient(t, bj, bc, flatSourceSecret(), unowned)
|
|
r := &BackupJobReconciler{Client: c, CredentialsConfig: defaultCfg()}
|
|
if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}}); err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
// BackupJob must remain non-terminal: foreign APIGroup short-circuits
|
|
// without any phase transition.
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}, got); err != nil {
|
|
t.Fatalf("fetch BackupJob: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("foreign-APIGroup BackupJob was terminally failed: %+v", got.Status)
|
|
}
|
|
// The pre-existing unowned Secret must be untouched.
|
|
gotSec := &corev1.Secret{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, gotSec); err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if string(gotSec.Data["user-data"]) != "x" {
|
|
t.Fatalf("unowned Secret mutated by foreign-APIGroup reconcile: %v", gotSec.Data)
|
|
}
|
|
if _, leaked := gotSec.Data["AWS_ACCESS_KEY_ID"]; leaked {
|
|
t.Fatalf("projector ran for foreign-APIGroup BackupJob; AWS creds leaked into unowned Secret")
|
|
}
|
|
}
|
|
|
|
// TestReconcile_NoMatchingStrategy_IsTerminal covers round-8 blocker #2:
|
|
// a BackupJob whose BackupClass does not bind a strategy for the
|
|
// applicationRef Kind (the documented FoundationDB-vs-cozy-default case)
|
|
// must transition to Phase=Failed with a clear message instead of being
|
|
// requeued forever by controller-runtime's default error-handling.
|
|
func TestReconcile_NoMatchingStrategy_IsTerminal(t *testing.T) {
|
|
appsGroup := "apps.cozystack.io"
|
|
bc := &backupsv1alpha1.BackupClass{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"},
|
|
Spec: backupsv1alpha1.BackupClassSpec{
|
|
Strategies: []backupsv1alpha1.BackupClassStrategy{
|
|
// Only Postgres is bound — FoundationDB intentionally omitted.
|
|
{
|
|
Application: backupsv1alpha1.ApplicationSelector{APIGroup: &appsGroup, Kind: "Postgres"},
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: stringRef(strategyv1alpha1.GroupVersion.Group), Kind: "CNPG", Name: "cozy-default-cnpg",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-acme", Name: "fdb-bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
BackupClassName: "cozy-default",
|
|
ApplicationRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &appsGroup, Kind: "FoundationDB", Name: "fdb",
|
|
},
|
|
},
|
|
}
|
|
c := newBackupJobTestClient(t, bj, bc)
|
|
r := &BackupJobReconciler{Client: c}
|
|
res, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "tenant-acme", Name: "fdb-bj"}})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 || res.Requeue {
|
|
t.Fatalf("expected terminal Failed with no requeue, got %+v", res)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "fdb-bj"}, got); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Phase=Failed, got %q", got.Status.Phase)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "does not bind a strategy") {
|
|
t.Errorf("expected misbinding message, got %q", got.Status.Message)
|
|
}
|
|
}
|
|
|
|
func stringRef(s string) *string { return &s }
|
|
|
|
// TestReconcile_BackupClassNotFound_IsTransient covers round-12 blocker
|
|
// #3: a BackupJob referencing a BackupClass that does not exist must
|
|
// requeue with Ready=False/BackupClassNotFound, not return a bare
|
|
// apiserver error that controller-runtime will log at Error level on
|
|
// every backoff cycle. The cozy-default BackupClass is gated on a
|
|
// populated BucketClaim status, so on a fresh-cluster bootstrap the
|
|
// BackupClass legitimately does not exist for a window — tenant
|
|
// BackupJobs filed during that window need a clear status signal.
|
|
func TestReconcile_BackupClassNotFound_IsTransient(t *testing.T) {
|
|
appsGroup := "apps.cozystack.io"
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-acme", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
BackupClassName: "cozy-default",
|
|
ApplicationRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &appsGroup, Kind: "Postgres", Name: "pg",
|
|
},
|
|
},
|
|
}
|
|
// No BackupClass seeded — that's the point.
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c}
|
|
res, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}})
|
|
if err != nil {
|
|
t.Fatalf("expected nil error (transient handled in-band), got %v", err)
|
|
}
|
|
if res.RequeueAfter == 0 {
|
|
t.Fatalf("expected requeue, got %+v", res)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}, got); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("BackupClassNotFound must NOT be terminal; got Phase=%q", got.Status.Phase)
|
|
}
|
|
cond := meta.FindStatusCondition(got.Status.Conditions, "Ready")
|
|
if cond == nil || cond.Reason != "BackupClassNotFound" {
|
|
t.Errorf("expected Ready=False/BackupClassNotFound, got %+v", cond)
|
|
}
|
|
}
|
|
|
|
// TestReconcile_UnsupportedKindInPlatformAPIGroup_IsTerminal covers
|
|
// round-9 blocker #6: a BackupClass that resolves to
|
|
// strategy.backups.cozystack.io/<unknown Kind> used to project
|
|
// cozy-backups-creds and then silently no-op in the dispatch switch,
|
|
// leaving the BackupJob phaseless and the tenant namespace polluted
|
|
// with a leaked Secret. The fix rejects unsupported Kinds BEFORE
|
|
// projection and marks the BackupJob terminally Failed.
|
|
func TestReconcile_UnsupportedKindInPlatformAPIGroup_IsTerminal(t *testing.T) {
|
|
appsGroup := "apps.cozystack.io"
|
|
platformGroup := strategyv1alpha1.GroupVersion.Group
|
|
bc := &backupsv1alpha1.BackupClass{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"},
|
|
Spec: backupsv1alpha1.BackupClassSpec{
|
|
Strategies: []backupsv1alpha1.BackupClassStrategy{
|
|
{
|
|
Application: backupsv1alpha1.ApplicationSelector{APIGroup: &appsGroup, Kind: "Postgres"},
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &platformGroup, Kind: "MadeUpKind", Name: "cozy-default-madeup",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-acme", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
BackupClassName: "cozy-default",
|
|
ApplicationRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &appsGroup, Kind: "Postgres", Name: "pg",
|
|
},
|
|
},
|
|
}
|
|
c := newBackupJobTestClient(t, bj, bc)
|
|
r := &BackupJobReconciler{Client: c}
|
|
if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}}); err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "bj"}, got); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Phase=Failed for unsupported Kind in platform APIGroup, got %q", got.Status.Phase)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "not supported") {
|
|
t.Errorf("expected unsupported-Kind message, got %q", got.Status.Message)
|
|
}
|
|
// Critical: projection must NOT have leaked a Secret into the tenant ns.
|
|
leaked := &corev1.Secret{}
|
|
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, leaked); err == nil {
|
|
t.Fatalf("unsupported-Kind BackupJob leaked cozy-backups-creds into tenant namespace")
|
|
}
|
|
}
|
|
|
|
// TestHandleProjectionError pins the transient-vs-terminal split for
|
|
// credentials-projection failures. SourceSecretMissing and APIError must
|
|
// requeue (so the first BackupJob after a fresh install does not get
|
|
// stuck terminally Failed before the Bucket controller produces the
|
|
// source Secret); SourceSecretMalformed and TargetSecretNotOwned must
|
|
// fail terminally (they are operator-visible misconfigurations that will
|
|
// not self-heal).
|
|
func TestHandleProjectionError(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
err error
|
|
wantRequeue bool
|
|
wantPhase backupsv1alpha1.BackupJobPhase
|
|
wantReason string
|
|
}{
|
|
{
|
|
name: "source missing requeues",
|
|
err: &ProjectionError{Reason: ReasonSourceMissing, Message: "src/x not found"},
|
|
wantRequeue: true,
|
|
wantReason: "CredentialsProjectionPending",
|
|
},
|
|
{
|
|
name: "api error requeues",
|
|
err: &ProjectionError{Reason: ReasonAPIError, Message: "apiserver hiccup"},
|
|
wantRequeue: true,
|
|
wantReason: "CredentialsProjectionPending",
|
|
},
|
|
{
|
|
name: "malformed source is terminal",
|
|
err: &ProjectionError{Reason: ReasonSourceMalformed, Message: "no accessKey"},
|
|
wantPhase: backupsv1alpha1.BackupJobPhaseFailed,
|
|
wantReason: "BackupFailed",
|
|
},
|
|
{
|
|
name: "unowned target is terminal",
|
|
err: &ProjectionError{Reason: ReasonTargetNotOwned, Message: "tenant owns it"},
|
|
wantPhase: backupsv1alpha1.BackupJobPhaseFailed,
|
|
wantReason: "BackupFailed",
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "bj"}}
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c}
|
|
res, err := r.handleProjectionError(context.Background(), bj, tc.err)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if tc.wantRequeue {
|
|
if res.RequeueAfter == 0 {
|
|
t.Fatalf("expected RequeueAfter > 0, got %+v", res)
|
|
}
|
|
if bj.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("transient projection failure must not set Phase=Failed (got %q)", bj.Status.Phase)
|
|
}
|
|
} else {
|
|
if res.RequeueAfter != 0 {
|
|
t.Fatalf("expected terminal failure with no requeue, got %+v", res)
|
|
}
|
|
if bj.Status.Phase != tc.wantPhase {
|
|
t.Fatalf("expected Phase=%q, got %q", tc.wantPhase, bj.Status.Phase)
|
|
}
|
|
}
|
|
ready := meta.FindStatusCondition(bj.Status.Conditions, "Ready")
|
|
if ready == nil {
|
|
t.Fatalf("Ready condition not set: %+v", bj.Status.Conditions)
|
|
}
|
|
if ready.Reason != tc.wantReason {
|
|
t.Errorf("Ready.Reason: got %q want %q", ready.Reason, tc.wantReason)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStrategyNotReadyDeadlineExceeded(t *testing.T) {
|
|
now := metav1.Now()
|
|
recent := metav1.NewTime(now.Add(-time.Minute))
|
|
stale := metav1.NewTime(now.Add(-StrategyNotReadyDeadline - time.Minute))
|
|
|
|
if strategyNotReadyDeadlineExceeded(nil) {
|
|
t.Error("nil StartedAt must not trip the deadline (first reconcile sets it)")
|
|
}
|
|
if strategyNotReadyDeadlineExceeded(&recent) {
|
|
t.Error("a job within the bootstrap-window grace period must stay transient")
|
|
}
|
|
if !strategyNotReadyDeadlineExceeded(&stale) {
|
|
t.Error("a job past StrategyNotReadyDeadline must trip the deadline")
|
|
}
|
|
}
|
|
|
|
// TestRequeueStrategyNotReady_BoundedByDeadline covers BLOCKER 3: a
|
|
// strategyRef.name that never resolves (e.g. a typo in a custom BackupClass)
|
|
// must fail closed once the bootstrap-window grace period elapses instead of
|
|
// requeuing forever. Within the window it stays transient (Ready=False /
|
|
// StrategyNotReady) and requeues; past the deadline it flips to terminal
|
|
// Phase=Failed with a message naming the missing strategy.
|
|
func TestRequeueStrategyNotReady_BoundedByDeadline(t *testing.T) {
|
|
t.Run("within grace period stays transient and requeues", func(t *testing.T) {
|
|
recent := metav1.NewTime(metav1.Now().Add(-time.Minute))
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &recent},
|
|
}
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c}
|
|
|
|
res, err := r.requeueStrategyNotReady(context.Background(), bj, "cozy-default-cnpg")
|
|
if err != nil {
|
|
t.Fatalf("requeueStrategyNotReady: %v", err)
|
|
}
|
|
if res.RequeueAfter != CredentialsProjectionRequeue {
|
|
t.Fatalf("expected requeue after %s, got %+v", CredentialsProjectionRequeue, res)
|
|
}
|
|
if bj.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("must not be terminal within the grace period")
|
|
}
|
|
ready := meta.FindStatusCondition(bj.Status.Conditions, "Ready")
|
|
if ready == nil || ready.Reason != "StrategyNotReady" {
|
|
t.Fatalf("expected Ready=False/StrategyNotReady, got %+v", ready)
|
|
}
|
|
})
|
|
|
|
t.Run("past deadline fails terminally naming the strategy", func(t *testing.T) {
|
|
stale := metav1.NewTime(metav1.Now().Add(-StrategyNotReadyDeadline - time.Minute))
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &stale},
|
|
}
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c}
|
|
|
|
res, err := r.requeueStrategyNotReady(context.Background(), bj, "cozy-default-cnpg")
|
|
if err != nil {
|
|
t.Fatalf("requeueStrategyNotReady: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Fatalf("terminal failure must not requeue, got %+v", res)
|
|
}
|
|
if bj.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Phase=Failed, got %q", bj.Status.Phase)
|
|
}
|
|
if !strings.Contains(bj.Status.Message, "cozy-default-cnpg") {
|
|
t.Errorf("failure message should name the missing strategy, got %q", bj.Status.Message)
|
|
}
|
|
})
|
|
}
|