mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 04:33:36 +09:00
kafkaRunDeadline silently substituted the 30m default for a backupTimeout that time.ParseDuration rejects or that falls below the one-minute floor, with no Event, condition or log line. An operator who set an unparseable or sub-minute value, then raised it, saw every run keep dying at DeadlineExceeded with nothing naming the discarded parameter. Return the applied deadline together with a message when the configured value is rejected, and record it as a BackupTimeoutInvalid Warning Event on the BackupJob and RestoreJob (the reconcilers already hold a Recorder). An honoured or unset value stays silent. TestKafkaRunDeadline now asserts the warning fires for the rejected cases and not otherwise. Signed-off-by: Andrey Kolkov <androndo@gmail.com> Assisted-by: LLM
914 lines
41 KiB
Go
914 lines
41 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
dynamicfake "k8s.io/client-go/dynamic/fake"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/kafkatypes"
|
|
)
|
|
|
|
func strp(s string) *string { return &s }
|
|
|
|
func TestValidateKafkaApplicationRef(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
ref corev1.TypedLocalObjectReference
|
|
wantErr bool
|
|
}{
|
|
{"kind Kafka, empty group", corev1.TypedLocalObjectReference{Kind: "Kafka"}, false},
|
|
{"kind Kafka, default group", corev1.TypedLocalObjectReference{Kind: "Kafka", APIGroup: strp("apps.cozystack.io")}, false},
|
|
{"wrong kind", corev1.TypedLocalObjectReference{Kind: "Postgres"}, true},
|
|
{"wrong group", corev1.TypedLocalObjectReference{Kind: "Kafka", APIGroup: strp("kafka.strimzi.io")}, true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := validateKafkaApplicationRef(tc.ref)
|
|
if tc.wantErr != (err != nil) {
|
|
t.Fatalf("validateKafkaApplicationRef(%+v) err=%v, wantErr=%v", tc.ref, err, tc.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestKafkaClusterName(t *testing.T) {
|
|
if got := kafkaClusterName("foo"); got != "kafka-foo" {
|
|
t.Fatalf("kafkaClusterName(foo) = %q, want kafka-foo", got)
|
|
}
|
|
}
|
|
|
|
func TestKafkaNotReadyMessage(t *testing.T) {
|
|
ready := &kafkatypes.Kafka{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "kafka-app"},
|
|
Status: kafkatypes.KafkaStatus{Conditions: []metav1.Condition{
|
|
{Type: kafkatypes.ConditionTypeReady, Status: metav1.ConditionTrue},
|
|
}},
|
|
}
|
|
if msg := kafkaNotReadyMessage(ready); msg != "" {
|
|
t.Fatalf("ready cluster: got %q, want empty", msg)
|
|
}
|
|
|
|
notReady := &kafkatypes.Kafka{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "kafka-app"},
|
|
Status: kafkatypes.KafkaStatus{Conditions: []metav1.Condition{
|
|
{Type: kafkatypes.ConditionTypeReady, Status: metav1.ConditionFalse, Message: "brokers pending"},
|
|
}},
|
|
}
|
|
msg := kafkaNotReadyMessage(notReady)
|
|
if !strings.Contains(msg, "Ready=False") || !strings.Contains(msg, "brokers pending") {
|
|
t.Fatalf("not-ready cluster: got %q, want Ready=False + message", msg)
|
|
}
|
|
|
|
missing := &kafkatypes.Kafka{ObjectMeta: metav1.ObjectMeta{Name: "kafka-app"}}
|
|
if msg := kafkaNotReadyMessage(missing); !strings.Contains(msg, "no Ready condition") {
|
|
t.Fatalf("missing condition: got %q, want 'no Ready condition'", msg)
|
|
}
|
|
}
|
|
|
|
func TestKafkaStrategyParameters_RoundTrip(t *testing.T) {
|
|
b := &backupsv1alpha1.Backup{
|
|
Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{
|
|
kafkaStrategyParamPrefix + "topics": "orders",
|
|
kafkaStrategyParamPrefix + "": "dropped-empty-key",
|
|
"unrelated": "ignored",
|
|
}},
|
|
}
|
|
got := kafkaStrategyParameters(b)
|
|
if len(got) != 1 || got["topics"] != "orders" {
|
|
t.Fatalf("kafkaStrategyParameters = %v, want {topics: orders}", got)
|
|
}
|
|
}
|
|
|
|
func TestRenderKafkaArtifactURI(t *testing.T) {
|
|
tmpl := "s3://bkt/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/kafka-topics.json"
|
|
|
|
ctxA := kafkaRenderContext(nil, "kafka-test", "tenant-root", kafkaStrategyModeBackup, "run-a", "", "", nil, nil)
|
|
uriA, err := renderKafkaArtifactURI(tmpl, ctxA)
|
|
if err != nil {
|
|
t.Fatalf("renderKafkaArtifactURI: %v", err)
|
|
}
|
|
if uriA != "s3://bkt/tenant-root/kafka-test/run-a/kafka-topics.json" {
|
|
t.Fatalf("uriA = %q", uriA)
|
|
}
|
|
|
|
ctxB := kafkaRenderContext(nil, "kafka-test", "tenant-root", kafkaStrategyModeBackup, "run-b", "", "", nil, nil)
|
|
uriB, _ := renderKafkaArtifactURI(tmpl, ctxB)
|
|
if uriA == uriB {
|
|
t.Fatalf("distinct BackupName must yield distinct key: %q == %q", uriA, uriB)
|
|
}
|
|
|
|
if uri, err := renderKafkaArtifactURI("", ctxA); err != nil || uri != "" {
|
|
t.Fatalf("empty template: uri=%q err=%v, want empty/nil", uri, err)
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_RejectsWrongKind covers the applicationRef Kind gate on the
|
|
// BackupJob path: a non-Kafka application terminates the BackupJob as Failed
|
|
// rather than running a Job against it.
|
|
func TestReconcileKafka_RejectsWrongKind(t *testing.T) {
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: strp("apps.cozystack.io"), Kind: "Postgres", Name: "pg",
|
|
},
|
|
BackupClassName: "cozy-default",
|
|
},
|
|
}
|
|
c := newBackupJobTestClient(t, bj)
|
|
r := &BackupJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka returned error: %v", err)
|
|
}
|
|
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("get bj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("wrong-kind BackupJob phase = %q, want Failed", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
func newKafkaApp(name, namespace string) *unstructured.Unstructured {
|
|
u := &unstructured.Unstructured{}
|
|
u.SetGroupVersionKind(schema.GroupVersionKind{
|
|
Group: backupsv1alpha1.DefaultApplicationAPIGroup,
|
|
Version: "v1alpha1",
|
|
Kind: "Kafka",
|
|
})
|
|
u.SetName(name)
|
|
u.SetNamespace(namespace)
|
|
return u
|
|
}
|
|
|
|
func newKafkaTestEnv(t *testing.T, app *unstructured.Unstructured, builder *clientfake.ClientBuilder) (*BackupJobReconciler, *RestoreJobReconciler) {
|
|
t.Helper()
|
|
testScheme := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(testScheme)
|
|
_ = backupsv1alpha1.AddToScheme(testScheme)
|
|
_ = strategyv1alpha1.AddToScheme(testScheme)
|
|
_ = kafkatypes.AddToScheme(testScheme)
|
|
|
|
gvr := schema.GroupVersionResource{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Resource: "kafkas"}
|
|
dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
|
|
testScheme, map[schema.GroupVersionResource]string{gvr: "KafkaList"}, app)
|
|
restMapper := &mockRESTMapper{mapping: &meta.RESTMapping{
|
|
Resource: gvr, GroupVersionKind: app.GroupVersionKind(), Scope: meta.RESTScopeNamespace,
|
|
}}
|
|
c := builder.WithScheme(testScheme).
|
|
WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}).
|
|
Build()
|
|
return &BackupJobReconciler{Client: c, Interface: dynamicClient, RESTMapper: restMapper, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)},
|
|
&RestoreJobReconciler{Client: c, Interface: dynamicClient, RESTMapper: restMapper, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
|
|
}
|
|
|
|
func newKafkaStrategy(name string) *strategyv1alpha1.Kafka {
|
|
return &strategyv1alpha1.Kafka{
|
|
ObjectMeta: metav1.ObjectMeta{Name: name},
|
|
Spec: strategyv1alpha1.KafkaSpec{
|
|
ArtifactURITemplate: "s3://bkt/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/kafka-metadata.txt",
|
|
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Containers: []corev1.Container{{Name: "kafka-backup", Image: "{{ .ClientImage }}", Args: []string{"--mode={{ .Mode }}", "--uri={{ .ArtifactURI }}"}}},
|
|
}},
|
|
},
|
|
}
|
|
}
|
|
|
|
func kafkaAppRef(name string) corev1.TypedLocalObjectReference {
|
|
return corev1.TypedLocalObjectReference{APIGroup: strp(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: "Kafka", Name: name}
|
|
}
|
|
|
|
// TestReconcileKafka_WaitsForNotReadyCluster: while the Strimzi Kafka cluster is
|
|
// not Ready the BackupJob must stay out of Succeeded (Ready=False precondition,
|
|
// requeue) and create no batch Job — the precondition is one of the driver's
|
|
// three reasons to exist, and a mutation dropping it must fail here.
|
|
func TestReconcileKafka_WaitsForNotReadyCluster(t *testing.T) {
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
// Preset StartedAt so reconcile skips the first-pass bookkeeping requeue
|
|
// and reaches the readiness gate.
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
cluster := &kafkatypes.Kafka{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "kafka-src"},
|
|
Status: kafkatypes.KafkaStatus{Conditions: []metav1.Condition{
|
|
{Type: kafkatypes.ConditionTypeReady, Status: metav1.ConditionFalse, Reason: "Creating", Message: "brokers starting"},
|
|
}},
|
|
}
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"), cluster))
|
|
|
|
res, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
if res.RequeueAfter <= 0 {
|
|
t.Fatalf("expected a requeue while the cluster is not Ready, got %+v", res)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("get bj: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded {
|
|
t.Fatalf("BackupJob reached Succeeded while cluster not Ready")
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(context.Background(), jobs, client.InNamespace("tenant")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Fatalf("expected no batch Job while cluster not Ready, got %d", len(jobs.Items))
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafkaRestore_RejectsWrongTargetKind: a to-copy restore whose
|
|
// target is not a Kafka application terminates the RestoreJob as Failed.
|
|
func TestReconcileKafkaRestore_RejectsWrongTargetKind(t *testing.T) {
|
|
pgGroup := backupsv1alpha1.DefaultApplicationAPIGroup
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: "bk"},
|
|
TargetApplicationRef: &corev1.TypedLocalObjectReference{APIGroup: &pgGroup, Kind: "Postgres", Name: "pg"},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: kafkaAppRef("src"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj, backup)
|
|
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
if _, err := r.reconcileKafkaRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcileKafkaRestore: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("get rj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("wrong-target-kind RestoreJob phase = %q, want Failed", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
// restoreJobRunning builds a RestoreJob already past its first-reconcile
|
|
// bookkeeping (StartedAt preset), so a test reaches the gates below it.
|
|
func restoreJobRunning(name, namespace string, target *corev1.TypedLocalObjectReference) *backupsv1alpha1.RestoreJob {
|
|
now := metav1.Now()
|
|
return &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: "bk"},
|
|
TargetApplicationRef: target,
|
|
},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafkaRestore_ToCopy pins the to-copy happy path AND the
|
|
// target-field merge: the restore resolves the TARGET (targetApplicationRef),
|
|
// not the source. Only "dst" is registered, so reaching Succeeded proves the
|
|
// TargetApplicationRef.Name merge was observed - ignoring it would resolve the
|
|
// absent source "src" and fail.
|
|
func TestReconcileKafkaRestore_ToCopy(t *testing.T) {
|
|
rj := restoreJobRunning("rj", "tenant",
|
|
&corev1.TypedLocalObjectReference{APIGroup: strp(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: "Kafka", Name: "dst"})
|
|
backup := kafkaBackup("bk", "tenant") // source "src", artifact URI recorded
|
|
completed := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj-restore"},
|
|
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}},
|
|
}
|
|
_, r := newKafkaTestEnv(t, newKafkaApp("dst", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(rj, backup, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-dst", "tenant"), kafkaBrokerPod("kafka-dst", "tenant", "quay.io/strimzi/kafka:test"), completed))
|
|
|
|
if _, err := r.reconcileKafkaRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcileKafkaRestore: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("get rj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded {
|
|
t.Fatalf("to-copy restore phase = %q, want Succeeded (target merge unobserved?)", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafkaRestore_NoRecordedArtifactURI pins that a Backup with no
|
|
// recorded object fails FAST, before the readiness gate - not after waiting out
|
|
// the deadline and reporting a misleading readiness timeout. No cluster is
|
|
// seeded, so the old ordering would requeue instead of failing.
|
|
func TestReconcileKafkaRestore_NoRecordedArtifactURI(t *testing.T) {
|
|
rj := restoreJobRunning("rj", "tenant", nil)
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: kafkaAppRef("src"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
},
|
|
}
|
|
_, r := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(rj, backup, newKafkaStrategy("cozy-default-kafka")))
|
|
|
|
if _, err := r.reconcileKafkaRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcileKafkaRestore: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("get rj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("no-artifact restore phase = %q, want Failed fast (not a readiness timeout)", got.Status.Phase)
|
|
}
|
|
if !strings.Contains(got.Status.Message, "artifact URI") {
|
|
t.Fatalf("message = %q, want it to name the missing artifact URI", got.Status.Message)
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafkaRestore_TargetClusterNotReady pins that while the target
|
|
// Kafka cluster is not Ready the restore requeues and creates no Job.
|
|
func TestReconcileKafkaRestore_TargetClusterNotReady(t *testing.T) {
|
|
rj := restoreJobRunning("rj", "tenant", nil)
|
|
backup := kafkaBackup("bk", "tenant")
|
|
notReady := &kafkatypes.Kafka{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "kafka-src"},
|
|
Status: kafkatypes.KafkaStatus{Conditions: []metav1.Condition{
|
|
{Type: kafkatypes.ConditionTypeReady, Status: metav1.ConditionFalse, Reason: "Creating", Message: "brokers starting"},
|
|
}},
|
|
}
|
|
_, r := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(rj, backup, newKafkaStrategy("cozy-default-kafka"), notReady))
|
|
|
|
res, err := r.reconcileKafkaRestore(context.Background(), rj, backup)
|
|
if err != nil {
|
|
t.Fatalf("reconcileKafkaRestore: %v", err)
|
|
}
|
|
if res.RequeueAfter <= 0 {
|
|
t.Fatalf("expected a requeue while the target cluster is not Ready, got %+v", res)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("get rj: %v", err)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded {
|
|
t.Fatal("restore reached Succeeded while target cluster not Ready")
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(context.Background(), jobs, client.InNamespace("tenant")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Fatalf("expected no restore Job while cluster not Ready, got %d", len(jobs.Items))
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafkaRestore_TargetNotFound pins that a restore whose target
|
|
// application is not registered terminates as Failed rather than requeuing.
|
|
func TestReconcileKafkaRestore_TargetNotFound(t *testing.T) {
|
|
rj := restoreJobRunning("rj", "tenant", nil) // target defaults to the source "src"
|
|
backup := kafkaBackup("bk", "tenant")
|
|
// Seed a different app so the target "src" resolves NotFound.
|
|
_, r := newKafkaTestEnv(t, newKafkaApp("other", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(rj, backup, newKafkaStrategy("cozy-default-kafka")))
|
|
|
|
if _, err := r.reconcileKafkaRestore(context.Background(), rj, backup); err != nil {
|
|
t.Fatalf("reconcileKafkaRestore: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.RestoreJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(rj), got); err != nil {
|
|
t.Fatalf("get rj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
|
t.Fatalf("missing-target restore phase = %q, want Failed", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
// TestCreateKafkaBackupArtifact_StampsScope pins that every produced Backup
|
|
// carries strategy.backups.cozystack.io/scope=topic-metadata, so `kubectl
|
|
// describe backup` and any consumer see the narrow scope. Dropping the stamp
|
|
// turns this red.
|
|
func TestCreateKafkaBackupArtifact_StampsScope(t *testing.T) {
|
|
testScheme := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(testScheme)
|
|
_ = backupsv1alpha1.AddToScheme(testScheme)
|
|
c := clientfake.NewClientBuilder().WithScheme(testScheme).
|
|
WithStatusSubresource(&backupsv1alpha1.Backup{}).Build()
|
|
r := &BackupJobReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src")},
|
|
}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}
|
|
backup, err := r.createKafkaBackupArtifact(context.Background(), bj, resolved, "s3://bkt/tenant/src/bj/kafka-metadata.txt", "quay.io/strimzi/kafka:test@sha256:deadbeef")
|
|
if err != nil {
|
|
t.Fatalf("createKafkaBackupArtifact: %v", err)
|
|
}
|
|
if got := backup.Spec.DriverMetadata[kafkaScopeKey]; got != kafkaScopeValue {
|
|
t.Fatalf("scope stamp = %q, want %q (driverMetadata=%v)", got, kafkaScopeValue, backup.Spec.DriverMetadata)
|
|
}
|
|
// The resolved broker image is recorded so cleanup can reuse it once the
|
|
// source cluster is gone.
|
|
if got := backup.Spec.DriverMetadata[kafkaStrategyClientImageKey]; got != "quay.io/strimzi/kafka:test@sha256:deadbeef" {
|
|
t.Fatalf("recorded client image = %q, want the resolved broker image (driverMetadata=%v)", got, backup.Spec.DriverMetadata)
|
|
}
|
|
}
|
|
|
|
// TestCreateKafkaBackupArtifact_RejectsDifferentApp pins the adoption guard: a
|
|
// same-named Backup left over for a different application is not reported as this
|
|
// run's success (mirrors the RabbitMQ driver).
|
|
func TestCreateKafkaBackupArtifact_RejectsDifferentApp(t *testing.T) {
|
|
testScheme := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(testScheme)
|
|
_ = backupsv1alpha1.AddToScheme(testScheme)
|
|
|
|
// A retained Backup named "bj" that describes application "other".
|
|
stale := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: kafkaAppRef("other")},
|
|
}
|
|
c := clientfake.NewClientBuilder().WithScheme(testScheme).
|
|
WithStatusSubresource(&backupsv1alpha1.Backup{}).WithObjects(stale).Build()
|
|
r := &BackupJobReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src")},
|
|
}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}
|
|
if _, err := r.createKafkaBackupArtifact(context.Background(), bj, resolved, "s3://bkt/tenant/src/bj/kafka-metadata.txt", "quay.io/strimzi/kafka:test@sha256:deadbeef"); err == nil {
|
|
t.Fatal("expected an error adopting a same-named Backup for a different application, got nil")
|
|
} else if !strings.Contains(err.Error(), "different application") {
|
|
t.Fatalf("error = %v, want it to mention a different application", err)
|
|
}
|
|
}
|
|
|
|
// TestCleanupOnDelete_Kafka_RoutesToKafkaCleanup pins the Backup-delete dispatch
|
|
// case: a Kafka Backup must route to cleanupKafkaBackup (which spawns the delete
|
|
// Job and requeues), not fall through to the Velero default that would release
|
|
// the object un-deleted. Dropping the `case KafkaStrategyKind` turns this red.
|
|
func TestCleanupOnDelete_Kafka_RoutesToKafkaCleanup(t *testing.T) {
|
|
testScheme := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(testScheme)
|
|
_ = backupsv1alpha1.AddToScheme(testScheme)
|
|
_ = strategyv1alpha1.AddToScheme(testScheme)
|
|
|
|
backup := kafkaBackup("kafka-src", "tenant-test")
|
|
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}}
|
|
c := clientfake.NewClientBuilder().WithScheme(testScheme).
|
|
WithStatusSubresource(&batchv1.Job{}).
|
|
WithObjects(ns, backup, newKafkaStrategy("cozy-default-kafka")).Build()
|
|
r := &BackupReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
res, err := r.cleanupOnDelete(context.Background(), backup)
|
|
if err != nil {
|
|
t.Fatalf("cleanupOnDelete: %v", err)
|
|
}
|
|
if res.RequeueAfter == 0 {
|
|
t.Fatal("expected a requeue while the Kafka delete Job runs; a fall-through to the Velero default would release the object un-deleted")
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := c.List(context.Background(), jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 || jobs.Items[0].Name != "kafka-src-cleanup" {
|
|
t.Fatalf("expected one kafka-src-cleanup Job from the Kafka dispatch route, got %+v", jobs.Items)
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_FailsAfterReadinessDeadline pins the deadline: a BackupJob
|
|
// whose cluster never becomes Ready must terminate as Failed once
|
|
// kafkaDefaultBackupDeadline elapses, not requeue forever. Neutralising the
|
|
// deadline comparison turns this red.
|
|
func TestReconcileKafka_FailsAfterReadinessDeadline(t *testing.T) {
|
|
old := metav1.NewTime(time.Now().Add(-(kafkaDefaultBackupDeadline + time.Minute)))
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &old, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
// App present, but the Strimzi Kafka cluster CR absent → never Ready.
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka")))
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("get bj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Failed after the readiness deadline, got %q", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
func getKafkaJob(t *testing.T, c client.Client, namespace, name string) *batchv1.Job {
|
|
t.Helper()
|
|
job := &batchv1.Job{}
|
|
if err := c.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: name}, job); err != nil {
|
|
t.Fatalf("get job %s/%s: %v", namespace, name, err)
|
|
}
|
|
return job
|
|
}
|
|
|
|
func TestKafkaRunDeadline(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
params map[string]string
|
|
want time.Duration
|
|
wantWarn bool // a rejected value must surface a non-empty warning
|
|
}{
|
|
{"unset falls back to default", nil, kafkaDefaultBackupDeadline, false},
|
|
{"valid override honoured", map[string]string{kafkaBackupTimeoutParam: "2h"}, 2 * time.Hour, false},
|
|
{"below floor falls back and warns", map[string]string{kafkaBackupTimeoutParam: "10s"}, kafkaDefaultBackupDeadline, true},
|
|
{"unparseable falls back and warns", map[string]string{kafkaBackupTimeoutParam: "nope"}, kafkaDefaultBackupDeadline, true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, warn := kafkaRunDeadline(tc.params)
|
|
if got != tc.want {
|
|
t.Fatalf("kafkaRunDeadline(%v) = %s, want %s", tc.params, got, tc.want)
|
|
}
|
|
if (warn != "") != tc.wantWarn {
|
|
t.Fatalf("kafkaRunDeadline(%v) warn = %q, wantWarn %v", tc.params, warn, tc.wantWarn)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_BoundsRunWithActiveDeadline pins that the run is bounded at
|
|
// the Job layer: the backup Job the controller creates carries
|
|
// activeDeadlineSeconds, so Kubernetes fails an unschedulable or wedged Job as
|
|
// DeadlineExceeded and kills its pod before the script's final upload - rather
|
|
// than a controller wall-clock that marks the BackupJob Failed while the Job
|
|
// runs on and orphans its object. Dropping setKafkaJobDeadline turns this red.
|
|
func TestReconcileKafka_BoundsRunWithActiveDeadline(t *testing.T) {
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-src", "tenant"), kafkaBrokerPod("kafka-src", "tenant", "quay.io/strimzi/kafka:test")))
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
job := getKafkaJob(t, r.Client, "tenant", "bj-backup")
|
|
if job.Spec.ActiveDeadlineSeconds == nil {
|
|
t.Fatal("backup Job must carry activeDeadlineSeconds to bound the run at the Job layer")
|
|
}
|
|
if got, want := *job.Spec.ActiveDeadlineSeconds, int64(kafkaDefaultBackupDeadline/time.Second); got != want {
|
|
t.Fatalf("activeDeadlineSeconds = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_BackupTimeoutParameterOverridesDeadline pins that the
|
|
// backupTimeout BackupClass parameter widens the Job's activeDeadlineSeconds, so
|
|
// a cluster whose export legitimately runs long is not capped at the default.
|
|
func TestReconcileKafka_BackupTimeoutParameterOverridesDeadline(t *testing.T) {
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-src", "tenant"), kafkaBrokerPod("kafka-src", "tenant", "quay.io/strimzi/kafka:test")))
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
Parameters: map[string]string{kafkaBackupTimeoutParam: "2h"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
job := getKafkaJob(t, r.Client, "tenant", "bj-backup")
|
|
if job.Spec.ActiveDeadlineSeconds == nil || *job.Spec.ActiveDeadlineSeconds != int64((2*time.Hour)/time.Second) {
|
|
t.Fatalf("activeDeadlineSeconds = %v, want %d from backupTimeout=2h", job.Spec.ActiveDeadlineSeconds, int64((2*time.Hour)/time.Second))
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_FailsWhenJobFails pins that a backup Job reported Failed -
|
|
// which is how Kubernetes marks a run that exceeds activeDeadlineSeconds -
|
|
// terminates the BackupJob as Failed rather than requeuing in Running forever.
|
|
func TestReconcileKafka_FailsWhenJobFails(t *testing.T) {
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
failedJob := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-backup"},
|
|
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{
|
|
{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Reason: "DeadlineExceeded", Message: "Job was active longer than specified deadline"},
|
|
}},
|
|
}
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-src", "tenant"), kafkaBrokerPod("kafka-src", "tenant", "quay.io/strimzi/kafka:test"), failedJob))
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("get bj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("expected Failed once the Job reported Failed, got %q", got.Status.Phase)
|
|
}
|
|
}
|
|
|
|
// podListErrClient forces PodList to fail, to simulate a transient apiserver
|
|
// error while resolving the broker image.
|
|
type podListErrClient struct {
|
|
client.Client
|
|
err error
|
|
}
|
|
|
|
func (c podListErrClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
|
|
if _, ok := list.(*corev1.PodList); ok {
|
|
return c.err
|
|
}
|
|
return c.Client.List(ctx, list, opts...)
|
|
}
|
|
|
|
// TestReconcileKafka_RequeuesTransientBrokerImageError pins that a transient
|
|
// List error while resolving the broker image requeues (returns an error) rather
|
|
// than terminating the whole backup - only a Ready cluster with no listable
|
|
// broker is terminal.
|
|
func TestReconcileKafka_RequeuesTransientBrokerImageError(t *testing.T) {
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-src", "tenant")))
|
|
r.Client = podListErrClient{Client: r.Client, err: apierrors.NewInternalError(errors.New("apiserver unavailable"))}
|
|
|
|
_, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected a requeue error on a transient broker List failure")
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if gerr := r.Get(context.Background(), client.ObjectKeyFromObject(bj), got); gerr != nil {
|
|
t.Fatalf("get bj: %v", gerr)
|
|
}
|
|
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatal("a transient List error must not terminate the backup as Failed")
|
|
}
|
|
}
|
|
|
|
// kafkaBackup builds a Backup fixture that has already recorded its metadata
|
|
// object, for the cleanup path (whose delete is keyed on status.artifact.uri).
|
|
func kafkaBackup(name, namespace string) *backupsv1alpha1.Backup {
|
|
return &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: kafkaAppRef("src"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
DriverMetadata: map[string]string{
|
|
kafkaScopeKey: kafkaScopeValue,
|
|
kafkaStrategyClientImageKey: "quay.io/strimzi/kafka:test@sha256:deadbeef",
|
|
},
|
|
},
|
|
Status: backupsv1alpha1.BackupStatus{
|
|
Phase: backupsv1alpha1.BackupPhaseReady,
|
|
Artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bkt/" + namespace + "/src/" + name + "/kafka-metadata.txt"},
|
|
},
|
|
}
|
|
}
|
|
|
|
// TestCleanupKafkaBackup_SpawnsDeleteJob pins that deleting a Backup spawns an
|
|
// ownerless, self-cleaning Job that deletes the recorded object in cleanup mode
|
|
// (with the URI injected), so a retention-pruned Plan does not leak objects -
|
|
// the same contract the Backup cleanup dispatcher routes KafkaStrategyKind to.
|
|
func TestCleanupKafkaBackup_SpawnsDeleteJob(t *testing.T) {
|
|
testScheme := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(testScheme)
|
|
_ = backupsv1alpha1.AddToScheme(testScheme)
|
|
_ = strategyv1alpha1.AddToScheme(testScheme)
|
|
|
|
backup := kafkaBackup("kafka-src", "tenant-test")
|
|
strategy := newKafkaStrategy("cozy-default-kafka")
|
|
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}}
|
|
c := clientfake.NewClientBuilder().WithScheme(testScheme).
|
|
WithStatusSubresource(&batchv1.Job{}).
|
|
WithObjects(ns, backup, strategy).Build()
|
|
// Empty CredentialsConfig: ProjectBackupCredentials is a no-op when disabled.
|
|
r := &BackupReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
|
|
ctx := context.Background()
|
|
|
|
res, err := r.cleanupKafkaBackup(ctx, backup)
|
|
if err != nil {
|
|
t.Fatalf("cleanupKafkaBackup() error = %v", err)
|
|
}
|
|
if res.RequeueAfter == 0 {
|
|
t.Error("expected a requeue while the delete Job runs (must not orphan the object)")
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := c.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 cleanup Job, got %d", len(jobs.Items))
|
|
}
|
|
job := jobs.Items[0]
|
|
if job.Name != "kafka-src-cleanup" {
|
|
t.Errorf("job name = %q, want kafka-src-cleanup", job.Name)
|
|
}
|
|
if len(job.OwnerReferences) != 0 {
|
|
t.Errorf("cleanup Job must be ownerless so it survives Backup deletion, got %d ownerRefs", len(job.OwnerReferences))
|
|
}
|
|
if job.Labels[kafkaStrategyLabelMode] != kafkaStrategyModeCleanup {
|
|
t.Errorf("mode label = %q, want %q", job.Labels[kafkaStrategyLabelMode], kafkaStrategyModeCleanup)
|
|
}
|
|
if job.Spec.TTLSecondsAfterFinished == nil || job.Spec.ActiveDeadlineSeconds == nil {
|
|
t.Error("expected TTLSecondsAfterFinished + ActiveDeadlineSeconds on the cleanup Job")
|
|
}
|
|
args := job.Spec.Template.Spec.Containers[0].Args
|
|
foundMode, foundURI := false, false
|
|
for _, a := range args {
|
|
if a == "--mode=cleanup" {
|
|
foundMode = true
|
|
}
|
|
if a == "--uri="+backup.Status.Artifact.URI {
|
|
foundURI = true
|
|
}
|
|
}
|
|
if !foundMode {
|
|
t.Errorf("expected the pod rendered in cleanup mode (--mode=cleanup), got args %v", args)
|
|
}
|
|
if !foundURI {
|
|
t.Errorf("expected the recorded artifact URI injected into the cleanup pod, got args %v", args)
|
|
}
|
|
// Cleanup runs through the image the backup recorded (the source broker may be
|
|
// gone), injected via the .ClientImage placeholder.
|
|
if img := job.Spec.Template.Spec.Containers[0].Image; img != backup.Spec.DriverMetadata[kafkaStrategyClientImageKey] {
|
|
t.Errorf("cleanup Job image = %q, want the recorded client image %q", img, backup.Spec.DriverMetadata[kafkaStrategyClientImageKey])
|
|
}
|
|
|
|
// Once the delete Job completes, cleanup finishes (zero Result) and reaps the
|
|
// Job - so the Backup is only released after the object is gone.
|
|
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
|
|
if err := c.Status().Update(ctx, &job); err != nil {
|
|
t.Fatalf("update job status: %v", err)
|
|
}
|
|
res, err = r.cleanupKafkaBackup(ctx, backup)
|
|
if err != nil {
|
|
t.Fatalf("cleanupKafkaBackup() second call error = %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 || res.Requeue {
|
|
t.Errorf("expected cleanup to finish once the Job completed, got requeue %+v", res)
|
|
}
|
|
if err := c.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Errorf("expected the completed cleanup Job to be reaped, got %d", len(jobs.Items))
|
|
}
|
|
}
|
|
|
|
func readyKafkaCluster(name, namespace string) *kafkatypes.Kafka {
|
|
return &kafkatypes.Kafka{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
|
|
Status: kafkatypes.KafkaStatus{Conditions: []metav1.Condition{
|
|
{Type: kafkatypes.ConditionTypeReady, Status: metav1.ConditionTrue, Reason: "Ready"},
|
|
}},
|
|
}
|
|
}
|
|
|
|
// kafkaBrokerPod builds a Strimzi broker pod the way the driver locates it: by
|
|
// the cluster label, with the image on a container named "kafka".
|
|
func kafkaBrokerPod(cluster, namespace, image string) *corev1.Pod {
|
|
return &corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: namespace,
|
|
Name: cluster + "-broker-0",
|
|
Labels: map[string]string{kafkaBrokerClusterLabel: cluster},
|
|
},
|
|
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: kafkaBrokerContainerName, Image: image}}},
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_ResolvesBrokerImage pins that the backup Job runs the target
|
|
// broker's own image (resolved from a live broker pod), not a chart pin. Dropping
|
|
// the resolver leaves the container image the unrendered ".ClientImage"
|
|
// placeholder, turning this red.
|
|
func TestReconcileKafka_ResolvesBrokerImage(t *testing.T) {
|
|
const brokerImage = "quay.io/strimzi/kafka:0.45.1-kafka-3.9.1@sha256:abc"
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-src", "tenant"), kafkaBrokerPod("kafka-src", "tenant", brokerImage)))
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(context.Background(), jobs, client.InNamespace("tenant")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected one backup Job, got %d", len(jobs.Items))
|
|
}
|
|
if img := jobs.Items[0].Spec.Template.Spec.Containers[0].Image; img != brokerImage {
|
|
t.Fatalf("backup Job image = %q, want the resolved broker image %q", img, brokerImage)
|
|
}
|
|
}
|
|
|
|
// TestReconcileKafka_FailsWhenNoBrokerImage pins the fail-closed behaviour: a
|
|
// Ready cluster with no resolvable broker image terminates the BackupJob Failed
|
|
// with a legible reason and creates no Job, rather than running one with an
|
|
// empty/placeholder image.
|
|
func TestReconcileKafka_FailsWhenNoBrokerImage(t *testing.T) {
|
|
now := metav1.Now()
|
|
bj := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: kafkaAppRef("src"), BackupClassName: "cozy-default"},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
// Ready cluster, but no broker pod to resolve the image from.
|
|
r, _ := newKafkaTestEnv(t, newKafkaApp("src", "tenant"),
|
|
clientfake.NewClientBuilder().WithObjects(bj, newKafkaStrategy("cozy-default-kafka"),
|
|
readyKafkaCluster("kafka-src", "tenant")))
|
|
|
|
if _, err := r.reconcileKafka(context.Background(), bj, &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{Kind: strategyv1alpha1.KafkaStrategyKind, Name: "cozy-default-kafka"},
|
|
}); err != nil {
|
|
t.Fatalf("reconcileKafka: %v", err)
|
|
}
|
|
got := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(bj), got); err != nil {
|
|
t.Fatalf("get bj: %v", err)
|
|
}
|
|
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Fatalf("phase = %q, want Failed when the client image cannot be resolved", got.Status.Phase)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(context.Background(), jobs, client.InNamespace("tenant")); err != nil {
|
|
t.Fatalf("list jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Fatalf("expected no batch Job when the image cannot be resolved, got %d", len(jobs.Items))
|
|
}
|
|
}
|