mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-30 00:27:04 +09:00
Aleksei Sviridkin
520af57905
Private infrastructure names, a real hostname, a commented-out internal mirror block, an internal product name, and a reviewer's name in a comment do not belong in a public repository; drop them and replace what needs a stand-in with neutral placeholders. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
1887 lines
77 KiB
Go
1887 lines
77 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/client-go/dynamic"
|
|
dynamicfake "k8s.io/client-go/dynamic/fake"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/etcdapp"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/etcdtypes"
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func newEtcdTestScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(s)
|
|
_ = backupsv1alpha1.AddToScheme(s)
|
|
_ = strategyv1alpha1.AddToScheme(s)
|
|
_ = etcdapp.AddToScheme(s)
|
|
_ = etcdtypes.AddToScheme(s)
|
|
return s
|
|
}
|
|
|
|
func newEtcdTestClient(t *testing.T, objs ...client.Object) client.Client {
|
|
t.Helper()
|
|
return clientfake.NewClientBuilder().
|
|
WithScheme(newEtcdTestScheme(t)).
|
|
WithObjects(objs...).
|
|
WithStatusSubresource(
|
|
&backupsv1alpha1.BackupJob{},
|
|
&backupsv1alpha1.RestoreJob{},
|
|
&backupsv1alpha1.Backup{},
|
|
&etcdtypes.EtcdCluster{},
|
|
&etcdtypes.EtcdSnapshot{},
|
|
).
|
|
Build()
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Validation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestValidateEtcdApplicationRef(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
other := "other.example.com"
|
|
cases := []struct {
|
|
name string
|
|
ref corev1.TypedLocalObjectReference
|
|
wantErr bool
|
|
}{
|
|
{"happy path with apps group", corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "x", APIGroup: &apps}, false},
|
|
{"empty apiGroup accepted", corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "x"}, false},
|
|
{"foreign apiGroup rejected", corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "x", APIGroup: &other}, true},
|
|
{"wrong kind rejected", corev1.TypedLocalObjectReference{Kind: "Postgres", Name: "x", APIGroup: &apps}, true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := validateEtcdApplicationRef(tc.ref)
|
|
if tc.wantErr && err == nil {
|
|
t.Fatalf("expected validation error, got nil")
|
|
}
|
|
if !tc.wantErr && err != nil {
|
|
t.Fatalf("unexpected validation error: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateRenderedEtcdDestination(t *testing.T) {
|
|
good := strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
}
|
|
if err := validateRenderedEtcdDestination(good); err != nil {
|
|
t.Fatalf("good S3: %v", err)
|
|
}
|
|
|
|
cases := []struct {
|
|
name string
|
|
in strategyv1alpha1.EtcdDestinationTemplate
|
|
}{
|
|
// Empty destination - rejected because S3 is required (PVC is
|
|
// rejected at the CRD admission layer; this test only exercises
|
|
// the runtime defensive check for clients that bypass admission).
|
|
{"empty rejected", strategyv1alpha1.EtcdDestinationTemplate{}},
|
|
{"missing bucket", strategyv1alpha1.EtcdDestinationTemplate{S3: &strategyv1alpha1.EtcdS3Template{
|
|
Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
}}},
|
|
{"missing endpoint", strategyv1alpha1.EtcdDestinationTemplate{S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
}}},
|
|
{"missing credsRef name", strategyv1alpha1.EtcdDestinationTemplate{S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b", Endpoint: "https://e",
|
|
}}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if err := validateRenderedEtcdDestination(tc.in); err == nil {
|
|
t.Fatalf("expected error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestEtcdDestinationTemplate_NoPVCField pins the strategy CR API
|
|
// contract: PVC destinations are NOT exposed by this strategy. The
|
|
// upstream etcd-operator's PVC backup-write path
|
|
// (/backup/data/<subPath>/<backupName>.db with BACKUP_INCLUDE_REVISION
|
|
// / BACKUP_TIMESTAMP suffixes) and PVC restore-read path
|
|
// (/backup/data/<subPath> opened as a file, or /backup/data/snapshot.db
|
|
// when subPath is empty) disagree on the filename - a PVC backup taken
|
|
// via this strategy would be unrestoreable.
|
|
//
|
|
// Until upstream lands symmetric semantics (or surfaces the final
|
|
// on-disk path via the equivalent of BackupSnapshot.URI for PVC), the
|
|
// strategy CR rejects PVC at the CRD admission layer (XValidation
|
|
// rule "has(self.s3)") AND removes the Go-level field entirely so a
|
|
// future regression that re-adds it without fixing upstream fails
|
|
// this test instead of producing silently-unrestoreable backups.
|
|
//
|
|
// See review feedback in github.com/cozystack/cozystack PR #2641,
|
|
// review #4327696458, blocker #1.
|
|
func TestEtcdDestinationTemplate_NoPVCField(t *testing.T) {
|
|
typ := reflect.TypeFor[strategyv1alpha1.EtcdDestinationTemplate]()
|
|
if _, ok := typ.FieldByName("PVC"); ok {
|
|
t.Fatal("EtcdDestinationTemplate must not expose a PVC field; " +
|
|
"upstream PVC restore is broken (asymmetric path between " +
|
|
"backup-write and restore-read). Re-add only after upstream " +
|
|
"makes the paths symmetric; see PR #2641 review #4327696458.")
|
|
}
|
|
// Defense-in-depth: same for the snapshot persistence struct.
|
|
// snap.Destination embeds EtcdDestinationTemplate, so a re-added
|
|
// PVC field would also leak through Backup.status.underlyingResources.
|
|
snapType := reflect.TypeFor[etcdBackupSnapshot]()
|
|
destField, ok := snapType.FieldByName("Destination")
|
|
if !ok {
|
|
t.Fatal("etcdBackupSnapshot must have a Destination field")
|
|
}
|
|
if _, has := destField.Type.FieldByName("PVC"); has {
|
|
t.Fatal("etcdBackupSnapshot.Destination must not expose a PVC field")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Templating
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestRenderEtcdTemplate_AppName(t *testing.T) {
|
|
// The rendered template walks every string field and substitutes Go
|
|
// templates against .Application / .Parameters. Pin the canonical
|
|
// pattern (per-app credentials Secret name keyed off the app name)
|
|
// so a future refactor that drops templating from CredentialsSecretRef
|
|
// fails this test.
|
|
app := &etcdapp.Etcd{ObjectMeta: metav1.ObjectMeta{Name: "etcd-src", Namespace: "tenant-root"}}
|
|
in := strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "{{ .Parameters.bucket }}",
|
|
Endpoint: "https://s3.example.invalid",
|
|
Key: "{{ .Application.metadata.name }}/",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{
|
|
Name: "{{ .Application.metadata.name }}-etcd-backup-creds",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
rendered, err := renderEtcdTemplate(in, app, map[string]string{"bucket": "etcd-backups"})
|
|
if err != nil {
|
|
t.Fatalf("render: %v", err)
|
|
}
|
|
if rendered.Destination.S3.Bucket != "etcd-backups" {
|
|
t.Errorf("bucket: got %q want etcd-backups", rendered.Destination.S3.Bucket)
|
|
}
|
|
if rendered.Destination.S3.Key != "etcd-src/" {
|
|
t.Errorf("key: got %q want etcd-src/", rendered.Destination.S3.Key)
|
|
}
|
|
if rendered.Destination.S3.CredentialsSecretRef.Name != "etcd-src-etcd-backup-creds" {
|
|
t.Errorf("credentialsSecretRef.Name: got %q want etcd-src-etcd-backup-creds",
|
|
rendered.Destination.S3.CredentialsSecretRef.Name)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// strategyToEtcdBackupDestination shape-cast
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStrategyToEtcdBackupDestination_S3(t *testing.T) {
|
|
in := strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
Key: "etcd-src/",
|
|
Region: "us-east-1",
|
|
ForcePathStyle: new(true),
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
}
|
|
out, err := strategyToEtcdBackupDestination(in)
|
|
if err != nil {
|
|
t.Fatalf("convert: %v", err)
|
|
}
|
|
if out.S3 == nil || out.PVC != nil {
|
|
t.Fatalf("expected S3 set, PVC nil; got s3=%v pvc=%v", out.S3, out.PVC)
|
|
}
|
|
if out.S3.Bucket != "b" || out.S3.Endpoint != "https://e" || out.S3.Key != "etcd-src/" ||
|
|
out.S3.Region != "us-east-1" || out.S3.CredentialsSecretRef.Name != "creds" {
|
|
t.Errorf("shape cast: %+v", out.S3)
|
|
}
|
|
if out.S3.ForcePathStyle == nil || !*out.S3.ForcePathStyle {
|
|
t.Errorf("force path style not propagated: %v", out.S3.ForcePathStyle)
|
|
}
|
|
// Pin pointer-independence so a future refactor that aliases the
|
|
// strategy pointer into the operator type doesn't silently couple
|
|
// downstream mutations back into the strategy CR cache.
|
|
*in.S3.ForcePathStyle = false
|
|
if !*out.S3.ForcePathStyle {
|
|
t.Errorf("output should be independent of input pointer")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Snapshot encode/decode contract
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestEtcdBackupSnapshot_RoundTrip(t *testing.T) {
|
|
rendered := &strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
Key: "etcd-src/",
|
|
ForcePathStyle: new(true),
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
raw, err := marshalEtcdBackupSnapshot(rendered, map[string]string{"bucket": "etcd-backups"})
|
|
if err != nil {
|
|
t.Fatalf("marshal: %v", err)
|
|
}
|
|
snap, err := decodeEtcdBackupSnapshot(raw)
|
|
if err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if snap == nil || snap.Destination.S3 == nil || snap.Destination.S3.Bucket != "b" {
|
|
t.Fatalf("round-trip lost data: %+v", snap)
|
|
}
|
|
if snap.Parameters["bucket"] != "etcd-backups" {
|
|
t.Errorf("parameters not persisted: %v", snap.Parameters)
|
|
}
|
|
}
|
|
|
|
func TestEtcdBackupSnapshot_UnrecognisedKindIsTerminal(t *testing.T) {
|
|
bad := map[string]any{
|
|
"kind": "SomethingElse",
|
|
"apiVersion": etcdBackupSnapshotAPIVersion,
|
|
"destination": map[string]any{"s3": map[string]any{"bucket": "b", "endpoint": "e", "credentialsSecretRef": map[string]any{"name": "c"}}},
|
|
}
|
|
raw, _ := json.Marshal(bad)
|
|
_, err := decodeEtcdBackupSnapshot(&runtime.RawExtension{Raw: raw})
|
|
if !errors.Is(err, errEtcdSnapshotUnrecognised) {
|
|
t.Fatalf("expected errEtcdSnapshotUnrecognised, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestEtcdBackupSnapshot_MalformedJSONIsRecoverable(t *testing.T) {
|
|
_, err := decodeEtcdBackupSnapshot(&runtime.RawExtension{Raw: []byte("{not json")})
|
|
if err == nil || errors.Is(err, errEtcdSnapshotUnrecognised) {
|
|
t.Fatalf("expected recoverable decode error, got %v", err)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// resolveEtcdRestoreDestination: snapshot wins, driverMetadata falls back
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestResolveEtcdRestoreDestination_PrefersSnapshot(t *testing.T) {
|
|
snap := &etcdBackupSnapshot{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "from-snap",
|
|
Endpoint: "https://snap",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds-snap"},
|
|
},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "from-md",
|
|
etcdBackupEndpointKey: "https://md",
|
|
etcdBackupCredsSecretKey: "creds-md",
|
|
}}}
|
|
d, ok := resolveEtcdRestoreDestination(backup, snap)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.Bucket != "from-snap" || d.S3.CredentialsSecretRef.Name != "creds-snap" {
|
|
t.Errorf("snapshot should win; got %+v", d.S3)
|
|
}
|
|
}
|
|
|
|
func TestResolveEtcdRestoreDestination_FallsBackToDriverMetadata(t *testing.T) {
|
|
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "from-md",
|
|
etcdBackupEndpointKey: "https://md",
|
|
etcdBackupCredsSecretKey: "creds-md",
|
|
etcdBackupForcePathStyleKey: "true",
|
|
}}}
|
|
d, ok := resolveEtcdRestoreDestination(backup, nil)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.Bucket != "from-md" || d.S3.CredentialsSecretRef.Name != "creds-md" {
|
|
t.Errorf("driverMetadata fallback incorrect: %+v", d.S3)
|
|
}
|
|
if d.S3.ForcePathStyle == nil || !*d.S3.ForcePathStyle {
|
|
t.Errorf("ForcePathStyle should parse as true: %v", d.S3.ForcePathStyle)
|
|
}
|
|
}
|
|
|
|
func TestResolveEtcdRestoreDestination_MissingEverythingFails(t *testing.T) {
|
|
if _, ok := resolveEtcdRestoreDestination(&backupsv1alpha1.Backup{}, nil); ok {
|
|
t.Fatalf("expected !ok for empty backup")
|
|
}
|
|
}
|
|
|
|
// TestResolveEtcdRestoreDestination_ForcePathStyleNilRoundTrip pins
|
|
// the round-trip invariant for the optional ForcePathStyle field:
|
|
// when a backup is taken with ForcePathStyle unset (nil), the restore
|
|
// destination must ALSO be nil — neither path should silently default
|
|
// it to true (matching the chart's legacy backup.forcePathStyle=true)
|
|
// nor to false. This catches a future refactor where the backup-side
|
|
// driverMetadata encoding flips to write "false" unconditionally, or
|
|
// the restore-side decoder defaults missing keys to true.
|
|
//
|
|
// Exercises both resolution paths (snapshot + driverMetadata fallback).
|
|
func TestResolveEtcdRestoreDestination_ForcePathStyleNilRoundTrip(t *testing.T) {
|
|
t.Run("snapshot path: nil stays nil", func(t *testing.T) {
|
|
snap := &etcdBackupSnapshot{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b", Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
// ForcePathStyle deliberately unset (nil).
|
|
},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{
|
|
DriverMetadata: map[string]string{etcdBackupNameKey: "bk"},
|
|
}}
|
|
d, ok := resolveEtcdRestoreDestination(backup, snap)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.ForcePathStyle != nil {
|
|
t.Errorf("ForcePathStyle must round-trip as nil; got %v", *d.S3.ForcePathStyle)
|
|
}
|
|
})
|
|
t.Run("driverMetadata path: absent key stays nil", func(t *testing.T) {
|
|
// No etcdBackupForcePathStyleKey in DriverMetadata - this is
|
|
// exactly what the BackupJob path writes when
|
|
// s.ForcePathStyle == nil (see createEtcdBackupArtifact).
|
|
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "from-md",
|
|
etcdBackupEndpointKey: "https://md",
|
|
etcdBackupCredsSecretKey: "creds-md",
|
|
// etcdBackupForcePathStyleKey deliberately absent.
|
|
}}}
|
|
d, ok := resolveEtcdRestoreDestination(backup, nil)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.ForcePathStyle != nil {
|
|
t.Errorf("ForcePathStyle must round-trip as nil when driverMetadata key is absent; got %v", *d.S3.ForcePathStyle)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestResolveEtcdRestoreDestination_FullS3KeyForRestore pins the operator
|
|
// filename convention. The backup-side persists the strategy-rendered
|
|
// prefix (e.g. "etcd/") because internal/controller/factory/backup_job.go
|
|
// appends "<backupName>.db" when writing. The restore-agent reads
|
|
// S3_KEY verbatim, so the driver must rebuild "<prefix>/<backupName>.db"
|
|
// at restore-resolve time. A regression here surfaces in production as
|
|
// "downloaded snapshot is empty (0 bytes)" - the agent fetches the
|
|
// directory marker for the prefix.
|
|
func TestResolveEtcdRestoreDestination_FullS3KeyForRestore(t *testing.T) {
|
|
t.Run("snapshot prefix gets backup-name appended", func(t *testing.T) {
|
|
snap := &etcdBackupSnapshot{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
Key: "etcd/",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{
|
|
DriverMetadata: map[string]string{etcdBackupNameKey: "etcd-bk-abc123"},
|
|
}}
|
|
d, ok := resolveEtcdRestoreDestination(backup, snap)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.Key != "etcd/etcd-bk-abc123.db" {
|
|
t.Errorf("S3 restore key: got %q want %q", d.S3.Key, "etcd/etcd-bk-abc123.db")
|
|
}
|
|
})
|
|
t.Run("driverMetadata fallback also gets the filename", func(t *testing.T) {
|
|
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupKeyKey: "etcd/",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
etcdBackupNameKey: "etcd-bk-def456",
|
|
}}}
|
|
d, ok := resolveEtcdRestoreDestination(backup, nil)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.Key != "etcd/etcd-bk-def456.db" {
|
|
t.Errorf("S3 restore key: got %q want %q", d.S3.Key, "etcd/etcd-bk-def456.db")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestBuildEtcdRestoreS3Key(t *testing.T) {
|
|
cases := []struct {
|
|
prefix string
|
|
name string
|
|
want string
|
|
}{
|
|
{"", "bk", "bk.db"},
|
|
{"etcd/", "bk", "etcd/bk.db"},
|
|
{"etcd", "bk", "etcd/bk.db"},
|
|
{"etcd/backups/", "bk", "etcd/backups/bk.db"},
|
|
{"etcd/", "", "etcd/"}, // empty backup-name → prefix verbatim (legacy artefact)
|
|
}
|
|
for _, tc := range cases {
|
|
if got := buildEtcdRestoreS3Key(tc.prefix, tc.name); got != tc.want {
|
|
t.Errorf("buildEtcdRestoreS3Key(%q, %q): got %q want %q", tc.prefix, tc.name, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestS3KeyFromArtifactURI pins the parser that pulls the FULL S3 key
|
|
// from the operator-reported artifact URI. The driver prefers this
|
|
// over buildEtcdRestoreS3Key because BACKUP_INCLUDE_REVISION /
|
|
// BACKUP_TIMESTAMP env vars (set in operator v0.4.4 defaults) make
|
|
// the agent append "-rev<N>" / "-<timestamp>" to the filename at
|
|
// write time — values buildEtcdRestoreS3Key can NEVER reconstruct
|
|
// from spec alone. Without this path the restore-agent 404s on the
|
|
// suffix-less key the driver would otherwise emit.
|
|
func TestS3KeyFromArtifactURI(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
artifact *backupsv1alpha1.BackupArtifact
|
|
bucket string
|
|
wantKey string
|
|
wantOK bool
|
|
}{
|
|
{
|
|
name: "agent-emitted -rev2 suffix round-trips verbatim",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bucket-abc/etcd/etcd-backup-job-xyz-rev2.db"},
|
|
bucket: "bucket-abc",
|
|
wantKey: "etcd/etcd-backup-job-xyz-rev2.db",
|
|
wantOK: true,
|
|
},
|
|
{
|
|
name: "agent-emitted timestamp suffix round-trips",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bucket-abc/etcd/bk-20260519T164740Z.db"},
|
|
bucket: "bucket-abc",
|
|
wantKey: "etcd/bk-20260519T164740Z.db",
|
|
wantOK: true,
|
|
},
|
|
{
|
|
name: "no prefix, root-level object",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bucket-abc/bk-rev1.db"},
|
|
bucket: "bucket-abc",
|
|
wantKey: "bk-rev1.db",
|
|
wantOK: true,
|
|
},
|
|
{
|
|
name: "nil artifact → no",
|
|
artifact: nil,
|
|
bucket: "bucket-abc",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "empty URI → no",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: ""},
|
|
bucket: "bucket-abc",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "non-s3 scheme (PVC file://) → no",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "file:///backup/data/bk.db"},
|
|
bucket: "bucket-abc",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "URI with different bucket → no (safety belt)",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://other-bucket/etcd/bk.db"},
|
|
bucket: "bucket-abc",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "URI with no key, trailing slash → no",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bucket-abc/"},
|
|
bucket: "bucket-abc",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "URI with no slash after bucket → no",
|
|
artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bucket-abc"},
|
|
bucket: "bucket-abc",
|
|
wantOK: false,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, ok := s3KeyFromArtifactURI(tc.artifact, tc.bucket)
|
|
if ok != tc.wantOK {
|
|
t.Fatalf("ok=%v want %v (got=%q)", ok, tc.wantOK, got)
|
|
}
|
|
if ok && got != tc.wantKey {
|
|
t.Errorf("key=%q want %q", got, tc.wantKey)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestResolveEtcdRestoreDestination_PrefersArtifactURIForRevSuffix is
|
|
// the integration test for the bug found on a test cluster: with a v0.4.4
|
|
// operator (BACKUP_INCLUDE_REVISION=true), the agent wrote
|
|
// "etcd/etcd-backup-job-6gghq-rev2.db" but the driver's
|
|
// buildEtcdRestoreS3Key would emit "etcd/etcd-backup-job-6gghq.db" —
|
|
// the restore-agent 404s. The fix: prefer Backup.status.artifact.uri.
|
|
// This test fails on the pre-fix code and passes after.
|
|
func TestResolveEtcdRestoreDestination_PrefersArtifactURIForRevSuffix(t *testing.T) {
|
|
snap := &etcdBackupSnapshot{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "bucket-37bfe98a",
|
|
Endpoint: "https://s3.example",
|
|
Key: "etcd/", // suffix-less prefix that buildEtcdRestoreS3Key would round-trip
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
DriverMetadata: map[string]string{etcdBackupNameKey: "etcd-backup-job-6gghq"},
|
|
},
|
|
Status: backupsv1alpha1.BackupStatus{
|
|
// The operator-reported URI with the BACKUP_INCLUDE_REVISION suffix.
|
|
Artifact: &backupsv1alpha1.BackupArtifact{
|
|
URI: "s3://bucket-37bfe98a/etcd/etcd-backup-job-6gghq-rev2.db",
|
|
SizeBytes: 20512,
|
|
Checksum: "sha256:abc123",
|
|
},
|
|
},
|
|
}
|
|
d, ok := resolveEtcdRestoreDestination(backup, snap)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.Key != "etcd/etcd-backup-job-6gghq-rev2.db" {
|
|
t.Errorf("expected agent-emitted URI to win over buildEtcdRestoreS3Key reconstruction; got key=%q", d.S3.Key)
|
|
}
|
|
}
|
|
|
|
// TestResolveEtcdRestoreDestination_FallsBackWhenArtifactBucketMismatches
|
|
// pins the safety belt: if a future agent or a corrupted Backup
|
|
// artifact reports a URI for a different bucket than the
|
|
// destination's, the driver MUST fall back to reconstruction rather
|
|
// than divert the restore to a wrong bucket. The bucket-match check
|
|
// in s3KeyFromArtifactURI is the load-bearing guard.
|
|
func TestResolveEtcdRestoreDestination_FallsBackWhenArtifactBucketMismatches(t *testing.T) {
|
|
snap := &etcdBackupSnapshot{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "bucket-correct",
|
|
Endpoint: "https://s3.example",
|
|
Key: "etcd/",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
DriverMetadata: map[string]string{etcdBackupNameKey: "etcd-bk-abc"},
|
|
},
|
|
Status: backupsv1alpha1.BackupStatus{
|
|
Artifact: &backupsv1alpha1.BackupArtifact{
|
|
URI: "s3://wrong-bucket/etcd/etcd-bk-abc-rev1.db",
|
|
},
|
|
},
|
|
}
|
|
d, ok := resolveEtcdRestoreDestination(backup, snap)
|
|
if !ok || d.S3 == nil {
|
|
t.Fatalf("expected S3 destination, got %+v ok=%v", d, ok)
|
|
}
|
|
if d.S3.Bucket != "bucket-correct" {
|
|
t.Errorf("destination bucket must come from snapshot, not artifact: got %q", d.S3.Bucket)
|
|
}
|
|
if d.S3.Key != "etcd/etcd-bk-abc.db" {
|
|
t.Errorf("expected fallback to buildEtcdRestoreS3Key on bucket mismatch; got key=%q", d.S3.Key)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// ensureEtcdBackup idempotency: same labels => reuse, not duplicate
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestEnsureEtcdBackup_LabelIdempotency(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj-1"},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t)}
|
|
|
|
rendered := &strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
|
|
first, err := r.ensureEtcdSnapshot(context.Background(), bj, rendered)
|
|
if err != nil {
|
|
t.Fatalf("first ensure: %v", err)
|
|
}
|
|
second, err := r.ensureEtcdSnapshot(context.Background(), bj, rendered)
|
|
if err != nil {
|
|
t.Fatalf("second ensure: %v", err)
|
|
}
|
|
if first.Name != second.Name {
|
|
t.Errorf("expected idempotent reuse; got %q vs %q", first.Name, second.Name)
|
|
}
|
|
|
|
// And there's exactly one EtcdBackup in the namespace.
|
|
list := &etcdtypes.EtcdSnapshotList{}
|
|
if err := c.List(context.Background(), list, client.InNamespace(bj.Namespace)); err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
if len(list.Items) != 1 {
|
|
t.Errorf("expected 1 EtcdBackup, got %d", len(list.Items))
|
|
}
|
|
}
|
|
|
|
func TestEnsureEtcdBackup_NoOwnerReference(t *testing.T) {
|
|
// Same contract as the FoundationDB driver: per-BackupJob CRs must
|
|
// NOT carry an OwnerReference back to the BackupJob, because a
|
|
// kubectl delete && kubectl apply with the same BackupJob name has
|
|
// to reuse the prior operator CR via OwningJob labels. An owner ref
|
|
// would let Kubernetes GC reap the operator CR on parent deletion
|
|
// and defeat the reuse contract.
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj-1", UID: "abc"},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t)}
|
|
|
|
rendered := &strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
eb, err := r.ensureEtcdSnapshot(context.Background(), bj, rendered)
|
|
if err != nil {
|
|
t.Fatalf("ensure: %v", err)
|
|
}
|
|
if len(eb.OwnerReferences) != 0 {
|
|
t.Errorf("EtcdBackup must not carry an OwnerReference; got %+v", eb.OwnerReferences)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// createEtcdBackupArtifact: pass-through from EtcdBackup.status.snapshot
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestCreateEtcdBackupArtifact_PassesSnapshotThrough pins the contract
|
|
// between this driver and the upstream etcd-operator change that
|
|
// introduced EtcdBackup.status.snapshot (URI / SizeBytes / Checksum).
|
|
// The cozystack Backup.status.artifact must mirror those fields, NOT
|
|
// synthesise a URI from the spec destination (the rendered destination
|
|
// is the prefix, not the final key — the agent rewrites the suffix at
|
|
// write time, see the upstream BackupSnapshot godoc).
|
|
func TestCreateEtcdBackupArtifact_PassesSnapshotThrough(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj-1"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t)}
|
|
|
|
eb := &etcdtypes.EtcdSnapshot{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj-1-abcde"},
|
|
Status: etcdtypes.EtcdSnapshotStatus{
|
|
Phase: etcdtypes.BackupPhaseComplete,
|
|
Artifact: &etcdtypes.EtcdSnapshotArtifact{
|
|
URI: "s3://my-bucket/etcd/bj-1-abcde-rev42.db",
|
|
SizeBytes: 20512,
|
|
Checksum: "sha256:abcd1234",
|
|
},
|
|
},
|
|
}
|
|
rendered := &strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "my-bucket",
|
|
Endpoint: "https://s3.example",
|
|
Key: "etcd/",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
|
|
Kind: "Etcd", Name: "etcd-strategy-default",
|
|
}}
|
|
|
|
backup, err := r.createEtcdBackupArtifact(context.Background(), bj, resolved, eb, rendered)
|
|
if err != nil {
|
|
t.Fatalf("createEtcdBackupArtifact: %v", err)
|
|
}
|
|
if backup.Status.Artifact == nil {
|
|
t.Fatal("status.artifact: got nil, want pass-through")
|
|
}
|
|
if backup.Status.Artifact.URI != eb.Status.Artifact.URI {
|
|
t.Errorf("artifact.URI: got %q want %q", backup.Status.Artifact.URI, eb.Status.Artifact.URI)
|
|
}
|
|
if backup.Status.Artifact.SizeBytes != eb.Status.Artifact.SizeBytes {
|
|
t.Errorf("artifact.SizeBytes: got %d want %d", backup.Status.Artifact.SizeBytes, eb.Status.Artifact.SizeBytes)
|
|
}
|
|
if backup.Status.Artifact.Checksum != eb.Status.Artifact.Checksum {
|
|
t.Errorf("artifact.Checksum: got %q want %q", backup.Status.Artifact.Checksum, eb.Status.Artifact.Checksum)
|
|
}
|
|
}
|
|
|
|
// TestCreateEtcdBackupArtifact_NoSnapshotLeavesArtifactNil pins the
|
|
// older-operator fallback path: when EtcdBackup.status.snapshot is nil
|
|
// (operator pre-dates the upstream feature), the driver must leave
|
|
// Backup.status.artifact unset. Synthesising a URI from spec.destination
|
|
// would point at the prefix, not the actual object the agent wrote —
|
|
// strictly worse than nothing.
|
|
func TestCreateEtcdBackupArtifact_NoSnapshotLeavesArtifactNil(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj-2"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t)}
|
|
|
|
eb := &etcdtypes.EtcdSnapshot{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj-2-xyz"},
|
|
Status: etcdtypes.EtcdSnapshotStatus{
|
|
Phase: etcdtypes.BackupPhaseComplete,
|
|
// Snapshot is nil — older operator version.
|
|
},
|
|
}
|
|
rendered := &strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{
|
|
S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b",
|
|
Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
},
|
|
},
|
|
}
|
|
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
|
|
Kind: "Etcd", Name: "etcd-strategy-default",
|
|
}}
|
|
|
|
backup, err := r.createEtcdBackupArtifact(context.Background(), bj, resolved, eb, rendered)
|
|
if err != nil {
|
|
t.Fatalf("createEtcdBackupArtifact: %v", err)
|
|
}
|
|
if backup.Status.Artifact != nil {
|
|
t.Errorf("status.artifact: got %+v, want nil (older operator without snapshot field)", backup.Status.Artifact)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// RestoreJob: to-copy rejection
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestReconcileEtcdRestore_RejectsToCopy(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
},
|
|
}
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: "bk"},
|
|
TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd-dst", APIGroup: &apps},
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Errorf("expected Failed, got %q", got.Status.Phase)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "to-copy") {
|
|
t.Errorf("message should explain the limitation; got %q", got.Status.Message)
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_RejectsForeignTargetAPIGroup pins the
|
|
// defence-in-depth gate on the TARGET ref's APIGroup. Without this
|
|
// gate a tenant could craft a RestoreJob with
|
|
// {Kind: "Etcd", Name: <source>, APIGroup: "other.example.com"}:
|
|
// the Name+Kind checks accept (name matches the Backup's source,
|
|
// Kind matches "Etcd"), resolveEtcdRestoreTarget silently drops the
|
|
// APIGroup, and the restore runs against the Cozystack Etcd app
|
|
// instead of whatever foreign CRD the tenant intended. Mirrors the
|
|
// source-side validateEtcdApplicationRef check.
|
|
func TestReconcileEtcdRestore_RejectsForeignTargetAPIGroup(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
other := "other.example.com"
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
},
|
|
}
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: "bk"},
|
|
// Same name as the Backup's source so Name gate accepts;
|
|
// same Kind so Kind gate accepts; only APIGroup differs.
|
|
TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &other},
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Errorf("expected Failed phase on foreign target APIGroup; got %q (msg=%q)", got.Status.Phase, got.Status.Message)
|
|
}
|
|
if !strings.Contains(got.Status.Message, other) {
|
|
t.Errorf("message should name the rejected APIGroup; got %q", got.Status.Message)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// RestoreJob options parsing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestParseEtcdRestoreOptions(t *testing.T) {
|
|
t.Run("empty options falls back to default deadline", func(t *testing.T) {
|
|
o, err := parseEtcdRestoreOptions(nil)
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
if got := o.effectiveRestoreDeadline(); got != etcdDefaultRestoreDeadline {
|
|
t.Errorf("default deadline: got %v want %v", got, etcdDefaultRestoreDeadline)
|
|
}
|
|
})
|
|
t.Run("malformed JSON is terminal", func(t *testing.T) {
|
|
_, err := parseEtcdRestoreOptions(&runtime.RawExtension{Raw: []byte("{nope")})
|
|
if err == nil {
|
|
t.Fatalf("expected parse error")
|
|
}
|
|
})
|
|
t.Run("override propagates", func(t *testing.T) {
|
|
o, err := parseEtcdRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"restoreTimeoutSeconds":120}`)})
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
if d := o.effectiveRestoreDeadline(); d.Seconds() != 120 {
|
|
t.Errorf("deadline override: got %v want 120s", d)
|
|
}
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// etcdClusterReady gates on the operator-side Ready condition
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestEtcdClusterReady(t *testing.T) {
|
|
t.Run("nil cluster is not ready", func(t *testing.T) {
|
|
if etcdClusterReady(nil) {
|
|
t.Error("nil ready?")
|
|
}
|
|
})
|
|
t.Run("no conditions is not ready", func(t *testing.T) {
|
|
c := &etcdtypes.EtcdCluster{}
|
|
if etcdClusterReady(c) {
|
|
t.Error("no-conditions ready?")
|
|
}
|
|
})
|
|
t.Run("Ready=False is not ready", func(t *testing.T) {
|
|
c := &etcdtypes.EtcdCluster{}
|
|
apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{
|
|
Type: etcdtypes.ClusterConditionAvailable, Status: metav1.ConditionFalse, Reason: "Bootstrapping",
|
|
})
|
|
if etcdClusterReady(c) {
|
|
t.Error("Ready=False reported ready")
|
|
}
|
|
})
|
|
t.Run("Ready=True is ready", func(t *testing.T) {
|
|
c := &etcdtypes.EtcdCluster{}
|
|
apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{
|
|
Type: etcdtypes.ClusterConditionAvailable, Status: metav1.ConditionTrue, Reason: "Ok",
|
|
})
|
|
if !etcdClusterReady(c) {
|
|
t.Error("Ready=True not reported ready")
|
|
}
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// BackupJob terminal paths
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestReconcileEtcd_RejectsWrongKind(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Postgres", Name: "x", APIGroup: &apps},
|
|
BackupClassName: "etcd-default",
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd-strategy-default", APIGroup: &apps},
|
|
}
|
|
if _, err := r.reconcileEtcd(context.Background(), bj, resolved); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Errorf("expected Failed, got %q (msg=%q)", got.Status.Phase, got.Status.Message)
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcd_StrategyMissingIsTransient pins the bootstrap-window
|
|
// contract: when the Strategy CR referenced by a (resolved) BackupClass
|
|
// is not (yet) present, the driver must surface a transient
|
|
// Ready=False/StrategyNotReady condition and requeue rather than
|
|
// terminate the BackupJob. Strategy CRs are gated on a populated
|
|
// BucketClaim.status.bucketName in the backupstrategy-controller chart;
|
|
// during the platform bootstrap window (Bucket → BucketClaim → COSI
|
|
// provisioning still in flight) BackupClass cozy-default exists but
|
|
// its strategies do not. Terminal failure would force tenants to
|
|
// recreate BackupJobs every time they raced the bootstrap.
|
|
func TestReconcileEtcd_StrategyMissingIsTransient(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
BackupClassName: "etcd-default",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
apiGroup := "strategy.backups.cozystack.io"
|
|
strategyRef := corev1.TypedLocalObjectReference{
|
|
Kind: "Etcd",
|
|
Name: "missing-strategy",
|
|
APIGroup: &apiGroup,
|
|
}
|
|
resolved := &ResolvedBackupConfig{StrategyRef: strategyRef}
|
|
res, err := r.reconcileEtcd(context.Background(), bj, resolved)
|
|
if err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
if res.RequeueAfter == 0 {
|
|
t.Errorf("expected RequeueAfter > 0, got %v", res.RequeueAfter)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Errorf("expected non-Failed phase, got %q", got.Status.Phase)
|
|
}
|
|
cond := apimeta.FindStatusCondition(got.Status.Conditions, "Ready")
|
|
if cond == nil {
|
|
t.Fatalf("expected Ready condition, got none")
|
|
}
|
|
if cond.Status != metav1.ConditionFalse || cond.Reason != "StrategyNotReady" {
|
|
t.Errorf("expected Ready=False/StrategyNotReady, got %s/%s", cond.Status, cond.Reason)
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcd_BackupJobDeadlineWhileWaitingForCluster pins the
|
|
// terminate-on-deadline contract for the wait-for-cluster branch:
|
|
// when the source EtcdCluster never becomes Ready (deleted source
|
|
// app, broken etcd, stuck PVC provisioner), the BackupJob must flip
|
|
// to Failed once StartedAt + etcdDefaultBackupDeadline elapses
|
|
// instead of requeuing forever. Mirrors the deadline check that
|
|
// already exists in the post-EtcdBackup-created path.
|
|
//
|
|
// Two sub-cases:
|
|
// 1. EtcdCluster NotFound + StartedAt long-past: Failed.
|
|
// 2. EtcdCluster present but Ready=False + StartedAt long-past: Failed.
|
|
//
|
|
// The third sub-case (StartedAt fresh) verifies the deadline doesn't
|
|
// fire prematurely.
|
|
func TestReconcileEtcd_BackupJobDeadlineWhileWaitingForCluster(t *testing.T) {
|
|
apps := etcdapp.GroupName
|
|
apiGroup := "strategy.backups.cozystack.io"
|
|
strategyRef := corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd-strategy-default", APIGroup: &apiGroup}
|
|
strategy := &strategyv1alpha1.Etcd{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "etcd-strategy-default"},
|
|
Spec: strategyv1alpha1.EtcdSpec{Template: strategyv1alpha1.EtcdTemplate{
|
|
Destination: strategyv1alpha1.EtcdDestinationTemplate{S3: &strategyv1alpha1.EtcdS3Template{
|
|
Bucket: "b", Endpoint: "https://e",
|
|
CredentialsSecretRef: strategyv1alpha1.EtcdLocalObjectReference{Name: "creds"},
|
|
}},
|
|
}},
|
|
}
|
|
app := &etcdapp.Etcd{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "etcd"}}
|
|
|
|
mkBJ := func(started time.Time) *backupsv1alpha1.BackupJob {
|
|
ts := metav1.NewTime(started)
|
|
return &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
BackupClassName: "etcd-default",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &ts},
|
|
}
|
|
}
|
|
resolved := &ResolvedBackupConfig{StrategyRef: strategyRef}
|
|
|
|
t.Run("EtcdCluster never appears -> Failed past deadline", func(t *testing.T) {
|
|
bj := mkBJ(time.Now().Add(-2 * etcdDefaultBackupDeadline))
|
|
c := newEtcdTestClient(t, bj, strategy, app)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
if _, err := r.reconcileEtcd(context.Background(), bj, resolved); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Failed phase, got %q (msg=%q)", got.Status.Phase, got.Status.Message)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "did not appear within") {
|
|
t.Errorf("failure message should mention deadline-on-NotFound; got %q", got.Status.Message)
|
|
}
|
|
})
|
|
|
|
t.Run("EtcdCluster present but Available=False -> Failed past deadline", func(t *testing.T) {
|
|
bj := mkBJ(time.Now().Add(-2 * etcdDefaultBackupDeadline))
|
|
notReady := &etcdtypes.EtcdCluster{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: etcdClusterName},
|
|
}
|
|
apimeta.SetStatusCondition(¬Ready.Status.Conditions, metav1.Condition{
|
|
Type: etcdtypes.ClusterConditionAvailable,
|
|
Status: metav1.ConditionFalse,
|
|
Reason: "Bootstrapping",
|
|
Message: "waiting for first quorum",
|
|
})
|
|
c := newEtcdTestClient(t, bj, strategy, app, notReady)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
if _, err := r.reconcileEtcd(context.Background(), bj, resolved); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Failed phase, got %q (msg=%q)", got.Status.Phase, got.Status.Message)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "did not become Available within") {
|
|
t.Errorf("failure message should mention Available-deadline; got %q", got.Status.Message)
|
|
}
|
|
// And the operator-side condition reason/message must be surfaced
|
|
// so an operator inspecting the failed BackupJob can see WHY the
|
|
// cluster never became Available.
|
|
if !strings.Contains(got.Status.Message, "waiting for first quorum") {
|
|
t.Errorf("failure message should surface operator Ready condition; got %q", got.Status.Message)
|
|
}
|
|
})
|
|
|
|
t.Run("StartedAt fresh -> requeue, not Failed", func(t *testing.T) {
|
|
bj := mkBJ(time.Now()) // just started
|
|
c := newEtcdTestClient(t, bj, strategy, app)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
res, err := r.reconcileEtcd(context.Background(), bj, resolved)
|
|
if err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
if res.RequeueAfter == 0 {
|
|
t.Fatalf("expected RequeueAfter > 0 while still within deadline window")
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("must not be Failed while still within deadline window; got Failed (msg=%q)", got.Status.Message)
|
|
}
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// latestEtcdSnapshotConditionMessage prefers failure-shaped conditions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestLatestEtcdBackupConditionMessage_PrefersFailedCondition pins the
|
|
// behaviour that a Failed/Ready=False condition wins over a later
|
|
// housekeeping update. Without this gate the upstream operator's
|
|
// post-failure status reconcile (e.g. flipping Started=False on the
|
|
// pod-gone path) would shadow the actual failure reason and tenants
|
|
// would see a misleading message on a Cozystack BackupJob.
|
|
func TestLatestEtcdBackupConditionMessage_PrefersFailedCondition(t *testing.T) {
|
|
failedAt := metav1.NewTime(time.Date(2026, 1, 1, 10, 0, 0, 0, time.UTC))
|
|
laterAt := metav1.NewTime(failedAt.Add(5 * time.Minute))
|
|
|
|
t.Run("Failed condition wins over later housekeeping", func(t *testing.T) {
|
|
eb := &etcdtypes.EtcdSnapshot{
|
|
Status: etcdtypes.EtcdSnapshotStatus{
|
|
Conditions: []metav1.Condition{
|
|
{Type: "Failed", Status: metav1.ConditionTrue, Reason: "UploadFailed",
|
|
Message: "upload to s3://x failed: AccessDenied", LastTransitionTime: failedAt},
|
|
{Type: "Started", Status: metav1.ConditionFalse, Reason: "JobGone",
|
|
Message: "job pod garbage collected", LastTransitionTime: laterAt},
|
|
},
|
|
},
|
|
}
|
|
got := latestEtcdSnapshotConditionMessage(eb)
|
|
if !strings.Contains(got, "AccessDenied") {
|
|
t.Errorf("expected failure message, got %q", got)
|
|
}
|
|
})
|
|
|
|
t.Run("Ready=False also counts as failure-shaped", func(t *testing.T) {
|
|
eb := &etcdtypes.EtcdSnapshot{
|
|
Status: etcdtypes.EtcdSnapshotStatus{
|
|
Conditions: []metav1.Condition{
|
|
{Type: etcdtypes.ClusterConditionAvailable, Status: metav1.ConditionFalse, Reason: "Backoff",
|
|
Message: "etcdctl exited 1: tls handshake timeout", LastTransitionTime: failedAt},
|
|
{Type: "Started", Status: metav1.ConditionFalse, Reason: "Idle",
|
|
Message: "idle", LastTransitionTime: laterAt},
|
|
},
|
|
},
|
|
}
|
|
got := latestEtcdSnapshotConditionMessage(eb)
|
|
if !strings.Contains(got, "tls handshake") {
|
|
t.Errorf("expected Ready=False message to be preferred, got %q", got)
|
|
}
|
|
})
|
|
|
|
t.Run("falls back to latest when no failure-shaped condition exists", func(t *testing.T) {
|
|
eb := &etcdtypes.EtcdSnapshot{
|
|
Status: etcdtypes.EtcdSnapshotStatus{
|
|
Conditions: []metav1.Condition{
|
|
{Type: "Started", Status: metav1.ConditionTrue, Reason: "Started",
|
|
Message: "snapshot in progress", LastTransitionTime: failedAt},
|
|
{Type: "Uploaded", Status: metav1.ConditionTrue, Reason: "Done",
|
|
Message: "uploaded 20512 bytes", LastTransitionTime: laterAt},
|
|
},
|
|
},
|
|
}
|
|
got := latestEtcdSnapshotConditionMessage(eb)
|
|
if !strings.Contains(got, "uploaded") {
|
|
t.Errorf("expected latest-by-time fallback, got %q", got)
|
|
}
|
|
})
|
|
|
|
t.Run("nil and empty are empty string", func(t *testing.T) {
|
|
if latestEtcdSnapshotConditionMessage(nil) != "" {
|
|
t.Error("nil should produce empty string")
|
|
}
|
|
if latestEtcdSnapshotConditionMessage(&etcdtypes.EtcdSnapshot{}) != "" {
|
|
t.Error("empty conditions should produce empty string")
|
|
}
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// requeueBackupJobWithReason flips phase to Running
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestRequeueBackupJobWithReason_SetsRunningPhase pins the contract that
|
|
// the first observable iteration (e.g. waiting for the source EtcdCluster
|
|
// to reach Ready) sets Phase=Running so tenants tailing
|
|
// BackupJob.status.phase see activity instead of an empty string while
|
|
// the condition spells out the wait reason.
|
|
func TestRequeueBackupJobWithReason_SetsRunningPhase(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
}
|
|
c := newEtcdTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Scheme: newEtcdTestScheme(t)}
|
|
if _, err := r.requeueBackupJobWithReason(context.Background(), bj, "EtcdClusterNotAvailable", "waiting"); err != nil {
|
|
t.Fatalf("requeue: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
|
|
t.Errorf("expected phase=Running on first observable iteration, got %q", got.Status.Phase)
|
|
}
|
|
cond := apimeta.FindStatusCondition(got.Status.Conditions, "Ready")
|
|
if cond == nil || cond.Reason != "EtcdClusterNotAvailable" {
|
|
t.Errorf("expected Ready=False/EtcdClusterNotAvailable condition, got %+v", cond)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// RestoreJob: captured-spec on Condition.Message size cap
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// testEtcdDynamicScheme returns a runtime.Scheme registered ONLY with
|
|
// the unstructured GVKs the restore path operates on through the
|
|
// dynamic client. Kept separate from newEtcdTestScheme (which carries
|
|
// the typed EtcdCluster) so the dynamic fake doesn't double-register
|
|
// the EtcdCluster GVK and panic.
|
|
func testEtcdDynamicScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(s)
|
|
s.AddKnownTypeWithName(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, &unstructured.Unstructured{})
|
|
s.AddKnownTypeWithName(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmReleaseList"}, &unstructured.UnstructuredList{})
|
|
s.AddKnownTypeWithName(schema.GroupVersionKind{Group: etcdtypes.GroupName, Version: etcdtypes.Version, Kind: "EtcdCluster"}, &unstructured.Unstructured{})
|
|
s.AddKnownTypeWithName(schema.GroupVersionKind{Group: etcdtypes.GroupName, Version: etcdtypes.Version, Kind: "EtcdClusterList"}, &unstructured.UnstructuredList{})
|
|
return s
|
|
}
|
|
|
|
// newOversizedEtcdCluster fabricates an unstructured EtcdCluster whose
|
|
// spec marshals to > etcdRestoreCapturedSpecMaxBytes. Used to exercise
|
|
// the fail-fast cap on Condition.Message overflow.
|
|
func newOversizedEtcdCluster(namespace string, bytes int) *unstructured.Unstructured {
|
|
bloat := make([]string, 0, bytes/16)
|
|
for len(bloat)*16 < bytes {
|
|
bloat = append(bloat, strings.Repeat("a", 15))
|
|
}
|
|
u := &unstructured.Unstructured{}
|
|
u.SetGroupVersionKind(schema.GroupVersionKind{Group: etcdtypes.GroupName, Version: etcdtypes.Version, Kind: "EtcdCluster"})
|
|
u.SetNamespace(namespace)
|
|
u.SetName(etcdClusterName)
|
|
_ = unstructured.SetNestedStringSlice(u.Object, bloat, "spec", "_bloat")
|
|
_ = unstructured.SetNestedField(u.Object, int64(3), "spec", "replicas")
|
|
return u
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_OversizedSpecFailsFastBeforePurge pins the
|
|
// fail-fast guard on Condition.Message overflow. If the captured spec
|
|
// would exceed etcdRestoreCapturedSpecMaxBytes the destructive purge
|
|
// MUST NOT start - the spec couldn't be durably persisted, so any crash
|
|
// between snapshot and recreate would leave the RestoreJob stuck with
|
|
// the cluster gone and the spec lost forever.
|
|
func TestReconcileEtcdRestore_OversizedSpecFailsFastBeforePurge(t *testing.T) {
|
|
const ns = "tenant"
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now},
|
|
}
|
|
|
|
cluster := newOversizedEtcdCluster(ns, etcdRestoreCapturedSpecMaxBytes*2)
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), cluster)
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Interface: dyn, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("expected Failed phase, got %q (msg=%q)", got.Status.Phase, got.Status.Message)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "exceeds") {
|
|
t.Errorf("failure message should explain the cap; got %q", got.Status.Message)
|
|
}
|
|
// The cluster must still exist - we must not have started the purge.
|
|
if _, err := dyn.Resource(etcdClusterGVR).Namespace(ns).Get(context.Background(), etcdClusterName, metav1.GetOptions{}); err != nil {
|
|
t.Errorf("oversized-spec gate fired AFTER deletion (cluster gone: %v); fail-fast contract broken", err)
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_EmptySpecFailsBeforePurge pins the empty-spec
|
|
// gate: a live EtcdCluster with an empty spec is the corrupted state the
|
|
// driver cannot recover from. The RestoreJob must terminate WITHOUT
|
|
// touching the cluster (deletion would lose the only reference to the
|
|
// chart-rendered fields needed for recreation).
|
|
func TestReconcileEtcdRestore_EmptySpecFailsBeforePurge(t *testing.T) {
|
|
const ns = "tenant"
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now},
|
|
}
|
|
|
|
empty := &unstructured.Unstructured{}
|
|
empty.SetGroupVersionKind(schema.GroupVersionKind{Group: etcdtypes.GroupName, Version: etcdtypes.Version, Kind: "EtcdCluster"})
|
|
empty.SetNamespace(ns)
|
|
empty.SetName(etcdClusterName)
|
|
// spec deliberately absent / empty.
|
|
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), empty)
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Interface: dyn, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("expected Failed phase on empty spec, got %q", got.Status.Phase)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "empty spec") {
|
|
t.Errorf("failure message should name the empty-spec condition; got %q", got.Status.Message)
|
|
}
|
|
if _, err := dyn.Resource(etcdClusterGVR).Namespace(ns).Get(context.Background(), etcdClusterName, metav1.GetOptions{}); err != nil {
|
|
t.Errorf("empty-spec gate must not delete the cluster; got: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_CrashRecoveryAdvancesWhenSpecCaptured pins the
|
|
// crash-safety branch: if the controller crashed AFTER capturing the
|
|
// spec on a Condition.Message and AFTER deleting the EtcdCluster but
|
|
// BEFORE stamping TargetPurged, the next reconcile sees NotFound on the
|
|
// dynamic Get. Without specCaptured we'd treat that as "tenant never
|
|
// deployed the source" and terminate; with specCaptured we treat it as
|
|
// "the previous iteration already purged" and advance the state machine.
|
|
func TestReconcileEtcdRestore_CrashRecoveryAdvancesWhenSpecCaptured(t *testing.T) {
|
|
const ns = "tenant"
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{
|
|
StartedAt: &now,
|
|
Conditions: []metav1.Condition{
|
|
{
|
|
Type: etcdRestoreCondClusterSpecCaptured,
|
|
Status: metav1.ConditionTrue,
|
|
Reason: "SpecCaptured",
|
|
Message: `{"replicas":3,"storage":{"volumeClaimTemplate":{}}}`,
|
|
LastTransitionTime: now,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t)) // no cluster -> NotFound
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Interface: dyn, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("expected reconcile to advance (NOT Failed); got Failed with msg=%q", got.Status.Message)
|
|
}
|
|
cond := apimeta.FindStatusCondition(got.Status.Conditions, etcdRestoreCondTargetPurged)
|
|
if cond == nil || cond.Status != metav1.ConditionTrue {
|
|
t.Errorf("expected TargetPurged=True (crash-recovery advance); got %+v", cond)
|
|
}
|
|
if cond != nil && cond.Reason != "ClusterPurgedRecovered" {
|
|
t.Errorf("expected Reason=ClusterPurgedRecovered to flag the crash-recovery path; got %q", cond.Reason)
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_MissingClusterWithoutCaptureIsTerminal pins
|
|
// the other side of the disambiguation: a tenant who fires a RestoreJob
|
|
// without ever having deployed the source app must NOT have the
|
|
// controller fall through to the recreate phase (which would create an
|
|
// EtcdCluster with an empty captured spec). The job must terminate.
|
|
func TestReconcileEtcdRestore_MissingClusterWithoutCaptureIsTerminal(t *testing.T) {
|
|
const ns = "tenant"
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now},
|
|
}
|
|
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t)) // no cluster + no capture
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Interface: dyn, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Errorf("expected Failed when source cluster never existed; got %q", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// HR-resume on terminal failure inside the destructive window
|
|
// (without this gate, helm-controller is frozen on the Etcd app forever)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// suspendedHelmRelease fabricates an unstructured HelmRelease with the
|
|
// requested spec.suspend value, under the GVK the driver mutates.
|
|
func suspendedHelmRelease(namespace, name string, suspend bool) *unstructured.Unstructured {
|
|
hr := &unstructured.Unstructured{}
|
|
hr.SetGroupVersionKind(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"})
|
|
hr.SetNamespace(namespace)
|
|
hr.SetName(name)
|
|
_ = unstructured.SetNestedMap(hr.Object, map[string]any{"suspend": suspend}, "spec")
|
|
return hr
|
|
}
|
|
|
|
func hrSuspended(t *testing.T, dyn dynamic.Interface, namespace, name string) bool {
|
|
t.Helper()
|
|
hr, err := dyn.Resource(helmReleaseGVR).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
|
|
if err != nil {
|
|
t.Fatalf("get HR: %v", err)
|
|
}
|
|
got, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend")
|
|
return got
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_AvailableDeadlineResumesHR pins the most likely
|
|
// production failure path: the recreated EtcdCluster does NOT become
|
|
// Available within restoreTimeoutSeconds (e.g. snapshot download stuck, S3
|
|
// creds wrong, PVC provisioner slow). Without this gate, the driver
|
|
// flips the RestoreJob Failed and leaves spec.suspend=true on the
|
|
// HelmRelease, freezing helm-controller on the Etcd app indefinitely —
|
|
// the only manual recovery is `kubectl patch hr/etcd -p
|
|
// '{"spec":{"suspend":false}}'`. It also pins the failure wording to the
|
|
// operator's actual condition (Available); the v1alpha2 operator emits no
|
|
// "Ready" condition.
|
|
//
|
|
// Setup walks the state machine to the Available-poll branch:
|
|
// - EtcdCluster exists (bootstrap.restore stamped, not Available),
|
|
// - pre-seeded TargetPurged AND EtcdClusterSpecCaptured so the
|
|
// destructive path is skipped on this reconcile,
|
|
// - HR is already suspended (simulating that a prior reconcile ran
|
|
// the destructive window),
|
|
// - StartedAt is far enough in the past that the deadline trips.
|
|
func TestReconcileEtcdRestore_AvailableDeadlineResumesHR(t *testing.T) {
|
|
const ns = "tenant"
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
},
|
|
},
|
|
}
|
|
long := metav1.NewTime(time.Now().Add(-2 * etcdDefaultRestoreDeadline))
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{
|
|
StartedAt: &long,
|
|
Conditions: []metav1.Condition{
|
|
{Type: etcdRestoreCondClusterSpecCaptured, Status: metav1.ConditionTrue, Reason: "SpecCaptured",
|
|
Message: `{"replicas":3,"storage":{"volumeClaimTemplate":{}}}`, LastTransitionTime: long},
|
|
{Type: etcdRestoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged",
|
|
Message: "purged", LastTransitionTime: long},
|
|
},
|
|
},
|
|
}
|
|
|
|
cluster := &unstructured.Unstructured{}
|
|
cluster.SetGroupVersionKind(schema.GroupVersionKind{Group: etcdtypes.GroupName, Version: etcdtypes.Version, Kind: "EtcdCluster"})
|
|
cluster.SetNamespace(ns)
|
|
cluster.SetName(etcdClusterName)
|
|
_ = unstructured.SetNestedField(cluster.Object, int64(3), "spec", "replicas")
|
|
|
|
hr := suspendedHelmRelease(ns, etcdClusterName, true)
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), cluster, hr)
|
|
|
|
// typed-client EtcdCluster (no Available=True condition) so the Available-poll branch trips the deadline.
|
|
typedCluster := &etcdtypes.EtcdCluster{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: etcdClusterName}}
|
|
c := newEtcdTestClient(t, backup, rj, typedCluster)
|
|
r := &RestoreJobReconciler{Client: c, Interface: dyn, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("expected Failed phase on Available deadline; got %q (msg=%q)", got.Status.Phase, got.Status.Message)
|
|
}
|
|
// The v1alpha2 operator emits no "Ready" condition; the restore-deadline
|
|
// message must report the actual gate (Available), not a condition that
|
|
// does not exist, or it misleads an operator debugging a stuck restore.
|
|
if !strings.Contains(got.Status.Message, "did not become Available within") {
|
|
t.Errorf("failure message should name the Available deadline; got %q", got.Status.Message)
|
|
}
|
|
if strings.Contains(got.Status.Message, "Ready") {
|
|
t.Errorf("failure message must not mention the nonexistent Ready condition; got %q", got.Status.Message)
|
|
}
|
|
if hrSuspended(t, dyn, ns, etcdClusterName) {
|
|
t.Fatalf("HelmRelease must NOT remain suspended after terminal Available-deadline failure - helm-controller would be frozen on the Etcd app")
|
|
}
|
|
}
|
|
|
|
// TestReconcileEtcdRestore_RecoverSpecErrorResumesHR pins the
|
|
// readCapturedEtcdClusterSpec failure branch: if the captured-spec
|
|
// condition is wiped (admission webhook corruption, manual edit, ...)
|
|
// after TargetPurged is set, the driver terminates the RestoreJob -
|
|
// and must resume the HR before doing so, because the destructive
|
|
// purge already happened on a prior iteration.
|
|
func TestReconcileEtcdRestore_RecoverSpecErrorResumesHR(t *testing.T) {
|
|
const ns = "tenant"
|
|
apps := etcdapp.GroupName
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "Etcd", Name: "etcd", APIGroup: &apps},
|
|
DriverMetadata: map[string]string{
|
|
etcdBackupBucketKey: "b",
|
|
etcdBackupEndpointKey: "https://e",
|
|
etcdBackupCredsSecretKey: "creds",
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{
|
|
StartedAt: &now,
|
|
// TargetPurged set but EtcdClusterSpecCaptured deliberately missing.
|
|
Conditions: []metav1.Condition{
|
|
{Type: etcdRestoreCondTargetPurged, Status: metav1.ConditionTrue, Reason: "ClusterPurged",
|
|
Message: "purged", LastTransitionTime: now},
|
|
},
|
|
},
|
|
}
|
|
|
|
hr := suspendedHelmRelease(ns, etcdClusterName, true)
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), hr) // no EtcdCluster -> recreate branch
|
|
c := newEtcdTestClient(t, backup, rj)
|
|
r := &RestoreJobReconciler{Client: c, Interface: dyn, Scheme: newEtcdTestScheme(t), Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileEtcdRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("refetch: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("expected Failed phase; got %q", got.Status.Phase)
|
|
}
|
|
if hrSuspended(t, dyn, ns, etcdClusterName) {
|
|
t.Fatalf("HelmRelease must be resumed on recover-spec failure; got suspended=true")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// etcdClusterFullyGone gates HR-resume until cluster + PVCs disappear
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestEtcdClusterFullyGone covers the wait-for-purge helper. Returning
|
|
// early on a still-terminating cluster lets the recreate step race the
|
|
// finalizer drain and the new Create races on PVC name collision.
|
|
func TestEtcdClusterFullyGone(t *testing.T) {
|
|
const ns = "tenant"
|
|
|
|
t.Run("cluster + PVCs absent -> gone=true", func(t *testing.T) {
|
|
c := newEtcdTestClient(t)
|
|
r := &RestoreJobReconciler{Client: c}
|
|
gone, err := r.etcdClusterFullyGone(context.Background(), ns)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if !gone {
|
|
t.Fatalf("expected gone=true with no cluster and no PVCs")
|
|
}
|
|
})
|
|
|
|
t.Run("live cluster -> gone=false", func(t *testing.T) {
|
|
cluster := &etcdtypes.EtcdCluster{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: etcdClusterName}}
|
|
c := newEtcdTestClient(t, cluster)
|
|
r := &RestoreJobReconciler{Client: c}
|
|
gone, err := r.etcdClusterFullyGone(context.Background(), ns)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if gone {
|
|
t.Fatalf("expected gone=false when cluster still exists")
|
|
}
|
|
})
|
|
|
|
t.Run("cluster gone but member PVC remains -> gone=false", func(t *testing.T) {
|
|
// The etcd-operator's StatefulSet-style volumeClaimTemplate names
|
|
// PVCs etcd-data-etcd-<i> and labels them with
|
|
// app.kubernetes.io/instance=etcd. Pin the label match: if the
|
|
// operator changes the label the restore will silently race the
|
|
// still-draining PVC.
|
|
pvc := &corev1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: ns,
|
|
Name: "etcd-data-etcd-0",
|
|
Labels: map[string]string{"app.kubernetes.io/instance": etcdClusterName},
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, pvc)
|
|
r := &RestoreJobReconciler{Client: c}
|
|
gone, err := r.etcdClusterFullyGone(context.Background(), ns)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if gone {
|
|
t.Fatalf("expected gone=false while member PVC still exists")
|
|
}
|
|
})
|
|
|
|
t.Run("terminating PVC (DeletionTimestamp set, finalizers draining) still blocks", func(t *testing.T) {
|
|
// The etcd-operator drains PVCs via finalizers / ownerRef cascade;
|
|
// a PVC mid-deletion has DeletionTimestamp set but stays in the
|
|
// apiserver until finalizers clear. If etcdClusterFullyGone
|
|
// returned gone=true on a terminating PVC, the recreate step
|
|
// would race the still-draining storage and the new
|
|
// EtcdCluster's volumeClaimTemplate Create would collide on PVC
|
|
// name. Pin that gone=false survives this state.
|
|
//
|
|
// Seeding pattern: the fake client preserves DeletionTimestamp
|
|
// when the seeded object carries both DeletionTimestamp AND a
|
|
// finalizer (without a finalizer the fake client treats the
|
|
// Delete as terminal). We use an etcd-operator-shaped finalizer
|
|
// name; the value is opaque to etcdClusterFullyGone.
|
|
now := metav1.Now()
|
|
pvc := &corev1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: ns,
|
|
Name: "etcd-data-etcd-0",
|
|
Labels: map[string]string{"app.kubernetes.io/instance": etcdClusterName},
|
|
DeletionTimestamp: &now,
|
|
Finalizers: []string{"kubernetes.io/pvc-protection"},
|
|
},
|
|
}
|
|
c := newEtcdTestClient(t, pvc)
|
|
r := &RestoreJobReconciler{Client: c}
|
|
gone, err := r.etcdClusterFullyGone(context.Background(), ns)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if gone {
|
|
t.Fatalf("expected gone=false while member PVC is still terminating (DeletionTimestamp set)")
|
|
}
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// setEtcdRestoreHRSuspended toggles spec.suspend on the HelmRelease
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestSetEtcdRestoreHRSuspended mirrors the CNPG driver's tests on the
|
|
// same helper. The destructive purge must suspend helm-controller before
|
|
// deleting the live EtcdCluster (otherwise Helm re-renders a
|
|
// bootstrap-less cluster on its next sync and races the restore) and
|
|
// resume it after the new cluster is Ready. A missing HR is treated as a
|
|
// no-op.
|
|
func TestSetEtcdRestoreHRSuspended(t *testing.T) {
|
|
const (
|
|
ns = "tenant"
|
|
name = "etcd"
|
|
)
|
|
mkHR := func(suspended *bool) *unstructured.Unstructured {
|
|
hr := &unstructured.Unstructured{}
|
|
hr.SetGroupVersionKind(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"})
|
|
hr.SetNamespace(ns)
|
|
hr.SetName(name)
|
|
spec := map[string]any{}
|
|
if suspended != nil {
|
|
spec["suspend"] = *suspended
|
|
}
|
|
_ = unstructured.SetNestedMap(hr.Object, spec, "spec")
|
|
return hr
|
|
}
|
|
suspendedField := func(t *testing.T, dyn dynamic.Interface) bool {
|
|
t.Helper()
|
|
hr, err := dyn.Resource(helmReleaseGVR).Namespace(ns).Get(context.Background(), name, metav1.GetOptions{})
|
|
if err != nil {
|
|
t.Fatalf("get HR: %v", err)
|
|
}
|
|
got, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend")
|
|
return got
|
|
}
|
|
|
|
t.Run("flips spec.suspend false -> true", func(t *testing.T) {
|
|
falsePtr := false
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), mkHR(&falsePtr))
|
|
r := &RestoreJobReconciler{Interface: dyn}
|
|
if err := r.setEtcdRestoreHRSuspended(context.Background(), ns, name, true); err != nil {
|
|
t.Fatalf("suspend: %v", err)
|
|
}
|
|
if !suspendedField(t, dyn) {
|
|
t.Fatalf("expected spec.suspend=true after suspend call")
|
|
}
|
|
})
|
|
|
|
t.Run("flips spec.suspend true -> false", func(t *testing.T) {
|
|
truePtr := true
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), mkHR(&truePtr))
|
|
r := &RestoreJobReconciler{Interface: dyn}
|
|
if err := r.setEtcdRestoreHRSuspended(context.Background(), ns, name, false); err != nil {
|
|
t.Fatalf("resume: %v", err)
|
|
}
|
|
if suspendedField(t, dyn) {
|
|
t.Fatalf("expected spec.suspend=false after resume call")
|
|
}
|
|
})
|
|
|
|
t.Run("idempotent when already at desired state", func(t *testing.T) {
|
|
truePtr := true
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t), mkHR(&truePtr))
|
|
r := &RestoreJobReconciler{Interface: dyn}
|
|
if err := r.setEtcdRestoreHRSuspended(context.Background(), ns, name, true); err != nil {
|
|
t.Fatalf("idempotent suspend: %v", err)
|
|
}
|
|
if !suspendedField(t, dyn) {
|
|
t.Fatalf("expected spec.suspend to stay true")
|
|
}
|
|
})
|
|
|
|
t.Run("missing HR returns nil (no-op)", func(t *testing.T) {
|
|
// A tenant deleting the Etcd app mid-restore must not strand the
|
|
// driver on a NotFound; the destructive flow should proceed and
|
|
// the next reconcile will note the missing HR.
|
|
dyn := dynamicfake.NewSimpleDynamicClient(testEtcdDynamicScheme(t))
|
|
r := &RestoreJobReconciler{Interface: dyn}
|
|
if err := r.setEtcdRestoreHRSuspended(context.Background(), ns, name, true); err != nil {
|
|
t.Fatalf("missing HR should be no-op: %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// EtcdRestoreOptions: negative override falls back to default deadline
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestEffectiveRestoreDeadline_NegativeFallsBack pins the contract on
|
|
// negative inputs. A tenant submitting restoreTimeoutSeconds=-1 (or any
|
|
// non-positive value) must get the default deadline rather than a
|
|
// zero/negative deadline that would mark the RestoreJob Failed on its
|
|
// very next poll.
|
|
func TestEffectiveRestoreDeadline_NegativeFallsBack(t *testing.T) {
|
|
for _, in := range []int64{-1, -3600, 0} {
|
|
o := EtcdRestoreOptions{RestoreTimeoutSeconds: in}
|
|
if got := o.effectiveRestoreDeadline(); got != etcdDefaultRestoreDeadline {
|
|
t.Errorf("RestoreTimeoutSeconds=%d: deadline=%v want %v", in, got, etcdDefaultRestoreDeadline)
|
|
}
|
|
}
|
|
}
|