cozystack/internal/backupcontroller/rabbitmqstrategy_controller_test.go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller

import (
	"context"
	"strings"
	"testing"
	"time"

	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	dynamicfake "k8s.io/client-go/dynamic/fake"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"

	strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
	backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
	"github.com/cozystack/cozystack/internal/backupcontroller/rabbitmqtypes"
)

func newRabbitmqApp(name, namespace string) *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   backupsv1alpha1.DefaultApplicationAPIGroup,
		Version: "v1alpha1",
		Kind:    rabbitmqAppKind,
	})
	u.SetName(name)
	u.SetNamespace(namespace)
	return u
}

func newRabbitmqRef(name string) corev1.TypedLocalObjectReference {
	return corev1.TypedLocalObjectReference{
		APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
		Kind:     rabbitmqAppKind,
		Name:     name,
	}
}

// newRabbitmqCluster builds the operator-side RabbitmqCluster the driver reads
// for its readiness gate. AllReplicasReady drives whether the precondition
// passes.
func newRabbitmqCluster(appName, namespace string, ready bool) *rabbitmqtypes.RabbitmqCluster {
	status := metav1.ConditionFalse
	if ready {
		status = metav1.ConditionTrue
	}
	return &rabbitmqtypes.RabbitmqCluster{
		ObjectMeta: metav1.ObjectMeta{Name: rabbitmqClusterName(appName), Namespace: namespace},
		Status: rabbitmqtypes.RabbitmqClusterStatus{
			Conditions: []metav1.Condition{{
				Type:               rabbitmqtypes.ConditionAllReplicasReady,
				Status:             status,
				Reason:             "Test",
				Message:            "test condition",
				LastTransitionTime: metav1.Now(),
			}},
		},
	}
}

func newRabbitmqStrategy(name string) *strategyv1alpha1.Rabbitmq {
	return &strategyv1alpha1.Rabbitmq{
		ObjectMeta: metav1.ObjectMeta{Name: name},
		Spec: strategyv1alpha1.RabbitmqSpec{
			ArtifactURITemplate: "s3://bkt/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/definitions.json",
			Template: corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					RestartPolicy: corev1.RestartPolicyNever,
					Containers: []corev1.Container{{
						Name:  "rabbitmq-backup",
						Image: "curl:test",
						Args: []string{
							"--host=rabbitmq-{{ .Release.Name }}.{{ .Release.Namespace }}.svc",
							"--mode={{ .Mode }}",
							"--uri={{ .ArtifactURI }}",
						},
					}},
				},
			},
		},
	}
}

func newRabbitmqResolved(strategyName string, params map[string]string) *ResolvedBackupConfig {
	return &ResolvedBackupConfig{
		StrategyRef: corev1.TypedLocalObjectReference{
			APIGroup: new(strategyv1alpha1.GroupVersion.Group),
			Kind:     strategyv1alpha1.RabbitmqStrategyKind,
			Name:     strategyName,
		},
		Parameters: params,
	}
}

func newRabbitmqTestEnv(t *testing.T, app *unstructured.Unstructured, builder *clientfake.ClientBuilder) (*BackupJobReconciler, *RestoreJobReconciler) {
	t.Helper()

	testScheme := runtime.NewScheme()
	_ = scheme.AddToScheme(testScheme)
	_ = backupsv1alpha1.AddToScheme(testScheme)
	_ = strategyv1alpha1.AddToScheme(testScheme)
	_ = rabbitmqtypes.AddToScheme(testScheme)

	gvr := schema.GroupVersionResource{
		Group:    backupsv1alpha1.DefaultApplicationAPIGroup,
		Version:  "v1alpha1",
		Resource: "rabbitmqs",
	}
	dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
		testScheme,
		map[schema.GroupVersionResource]string{gvr: rabbitmqAppKind + "List"},
		app,
	)
	mapping := &meta.RESTMapping{
		Resource:         gvr,
		GroupVersionKind: app.GroupVersionKind(),
		Scope:            meta.RESTScopeNamespace,
	}
	restMapper := &mockRESTMapper{mapping: mapping}

	c := builder.WithScheme(testScheme).
		// Backup deliberately omitted: its real CRD has no status subresource,
		// which is exactly why createRabbitmqBackupArtifact persists
		// status.artifact on Create. Registering it here would make the fake
		// drop status on Create and hide a regression the driver depends on.
		WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}).
		Build()

	return &BackupJobReconciler{
		Client:     c,
		Interface:  dynamicClient,
		RESTMapper: restMapper,
		Scheme:     testScheme,
		Recorder:   record.NewFakeRecorder(10),
	}, &RestoreJobReconciler{
		Client:     c,
		Interface:  dynamicClient,
		RESTMapper: restMapper,
		Scheme:     testScheme,
		Recorder:   record.NewFakeRecorder(10),
	}
}

func TestValidateRabbitmqApplicationRef(t *testing.T) {
	group := backupsv1alpha1.DefaultApplicationAPIGroup
	other := "example.com"
	tests := []struct {
		name    string
		ref     corev1.TypedLocalObjectReference
		wantErr bool
	}{
		{name: "RabbitMQ default group", ref: corev1.TypedLocalObjectReference{APIGroup: &group, Kind: "RabbitMQ"}},
		{name: "RabbitMQ empty group", ref: corev1.TypedLocalObjectReference{Kind: "RabbitMQ"}},
		{name: "wrong kind", ref: corev1.TypedLocalObjectReference{Kind: "Postgres"}, wantErr: true},
		{name: "wrong group", ref: corev1.TypedLocalObjectReference{APIGroup: &other, Kind: "RabbitMQ"}, wantErr: true},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			err := validateRabbitmqApplicationRef(tc.ref)
			if (err != nil) != tc.wantErr {
				t.Errorf("validateRabbitmqApplicationRef(%+v) err=%v, wantErr=%v", tc.ref, err, tc.wantErr)
			}
		})
	}
}

func TestRabbitmqClusterName(t *testing.T) {
	if got := rabbitmqClusterName("foo"); got != "rabbitmq-foo" {
		t.Errorf("rabbitmqClusterName(foo) = %q, want rabbitmq-foo", got)
	}
}

func TestRabbitmqNotReadyMessage(t *testing.T) {
	ready := &rabbitmqtypes.RabbitmqCluster{
		Status: rabbitmqtypes.RabbitmqClusterStatus{Conditions: []metav1.Condition{
			{Type: rabbitmqtypes.ConditionAllReplicasReady, Status: metav1.ConditionTrue},
		}},
	}
	if msg := rabbitmqNotReadyMessage(ready); msg != "" {
		t.Errorf("ready cluster: want empty message, got %q", msg)
	}

	notReady := &rabbitmqtypes.RabbitmqCluster{
		ObjectMeta: metav1.ObjectMeta{Name: "rabbitmq-x"},
		Status: rabbitmqtypes.RabbitmqClusterStatus{Conditions: []metav1.Condition{
			{Type: rabbitmqtypes.ConditionAllReplicasReady, Status: metav1.ConditionFalse, Message: "starting"},
		}},
	}
	if msg := rabbitmqNotReadyMessage(notReady); !strings.Contains(msg, "AllReplicasReady=False") || !strings.Contains(msg, "starting") {
		t.Errorf("not-ready cluster: unexpected message %q", msg)
	}

	missing := &rabbitmqtypes.RabbitmqCluster{ObjectMeta: metav1.ObjectMeta{Name: "rabbitmq-x"}}
	if msg := rabbitmqNotReadyMessage(missing); !strings.Contains(msg, "no AllReplicasReady") {
		t.Errorf("condition-less cluster: unexpected message %q", msg)
	}
}

func TestRenderRabbitmqArtifactURI(t *testing.T) {
	const tmpl = "s3://bkt/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/definitions.json"

	ctx1 := rabbitmqRenderContext(nil, "app-test", "tenant-test", rabbitmqModeBackup, "bj-1", "", nil, nil)
	uri1, err := renderRabbitmqArtifactURI(tmpl, ctx1)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if want := "s3://bkt/tenant-test/app-test/bj-1/definitions.json"; uri1 != want {
		t.Errorf("uri = %q, want %q", uri1, want)
	}

	// A second backup of the SAME application must render a DISTINCT key, or the
	// backups would overwrite each other in object storage and restoring an
	// older Backup would read the latest data.
	ctx2 := rabbitmqRenderContext(nil, "app-test", "tenant-test", rabbitmqModeBackup, "bj-2", "", nil, nil)
	uri2, err := renderRabbitmqArtifactURI(tmpl, ctx2)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if uri2 == uri1 {
		t.Errorf("two backups of one application produced the same key %q — they would overwrite each other", uri1)
	}

	if _, err := renderRabbitmqArtifactURI("{{ .Missing", ctx1); err == nil {
		t.Error("expected error for malformed artifact URI template")
	}
}

func TestRabbitmqBackupParameters_RoundTrip(t *testing.T) {
	backup := &backupsv1alpha1.Backup{
		Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{
			rabbitmqParamPrefix + "prefix": "p1",
			"other/key":                    "ignored",
			rabbitmqParamPrefix:            "ignored-empty-key",
		}},
	}
	got := rabbitmqBackupParameters(backup)
	if got["prefix"] != "p1" {
		t.Errorf("prefix = %q, want p1", got["prefix"])
	}
	if _, ok := got["other/key"]; ok {
		t.Errorf("non-parameter key leaked: %v", got)
	}
}

// TestReconcileRabbitmq_PreconditionGate pins that the readiness gate defers a
// run (Ready=False, no batch Job) until the RabbitmqCluster reports
// AllReplicasReady, then proceeds once it does.
func TestReconcileRabbitmq_PreconditionGate(t *testing.T) {
	newRunningBackupJob := func() *backupsv1alpha1.BackupJob {
		now := metav1.Now()
		return &backupsv1alpha1.BackupJob{
			ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
			Spec: backupsv1alpha1.BackupJobSpec{
				ApplicationRef:  newRabbitmqRef("rmq-test"),
				BackupClassName: "cozy-default",
			},
			Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
		}
	}

	t.Run("cluster not ready defers without creating a Job", func(t *testing.T) {
		app := newRabbitmqApp("rmq-test", "tenant-test")
		backupJob := newRunningBackupJob()
		cluster := newRabbitmqCluster("rmq-test", "tenant-test", false)
		r, _ := newRabbitmqTestEnv(t, app, clientfake.NewClientBuilder().
			WithObjects(backupJob, newRabbitmqStrategy("cozy-default-rabbitmq"), cluster))
		ctx := context.Background()

		res, err := r.reconcileRabbitmq(ctx, backupJob, newRabbitmqResolved("cozy-default-rabbitmq", nil))
		if err != nil {
			t.Fatalf("reconcileRabbitmq() error = %v", err)
		}
		if res.RequeueAfter == 0 {
			t.Error("expected a requeue while the cluster is not ready")
		}
		jobs := &batchv1.JobList{}
		if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
			t.Fatalf("list jobs: %v", err)
		}
		if len(jobs.Items) != 0 {
			t.Fatalf("expected no Job while the precondition is unmet, got %d", len(jobs.Items))
		}
		updated := &backupsv1alpha1.BackupJob{}
		if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
			t.Fatalf("get backupjob: %v", err)
		}
		cond := meta.FindStatusCondition(updated.Status.Conditions, "Ready")
		if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "RabbitMQClusterNotReady" {
			t.Errorf("expected Ready=False/RabbitMQClusterNotReady, got %+v", cond)
		}
	})

	t.Run("cluster ready proceeds to create the Job", func(t *testing.T) {
		app := newRabbitmqApp("rmq-test", "tenant-test")
		backupJob := newRunningBackupJob()
		cluster := newRabbitmqCluster("rmq-test", "tenant-test", true)
		r, _ := newRabbitmqTestEnv(t, app, clientfake.NewClientBuilder().
			WithObjects(backupJob, newRabbitmqStrategy("cozy-default-rabbitmq"), cluster))
		ctx := context.Background()

		if _, err := r.reconcileRabbitmq(ctx, backupJob, newRabbitmqResolved("cozy-default-rabbitmq", nil)); err != nil {
			t.Fatalf("reconcileRabbitmq() error = %v", err)
		}
		jobs := &batchv1.JobList{}
		if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
			t.Fatalf("list jobs: %v", err)
		}
		if len(jobs.Items) != 1 {
			t.Fatalf("expected the Job once the cluster is ready, got %d", len(jobs.Items))
		}
		args := jobs.Items[0].Spec.Template.Spec.Containers[0].Args
		want := []string{"--host=rabbitmq-rmq-test.tenant-test.svc", "--mode=backup"}
		for i, w := range want {
			if i >= len(args) || args[i] != w {
				t.Errorf("rendered args[%d]: want %q, got %q", i, w, args)
			}
		}
	})
}

// TestReconcileRabbitmq_RejectsWrongKind pins that a non-RabbitMQ applicationRef
// fails the BackupJob terminally rather than spawning a Job.
func TestReconcileRabbitmq_RejectsWrongKind(t *testing.T) {
	app := newRabbitmqApp("rmq-test", "tenant-test")
	backupJob := &backupsv1alpha1.BackupJob{
		ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
		Spec: backupsv1alpha1.BackupJobSpec{
			ApplicationRef:  corev1.TypedLocalObjectReference{APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: "Postgres", Name: "rmq-test"},
			BackupClassName: "cozy-default",
		},
	}
	r, _ := newRabbitmqTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob))
	ctx := context.Background()

	if _, err := r.reconcileRabbitmq(ctx, backupJob, newRabbitmqResolved("cozy-default-rabbitmq", nil)); err != nil {
		t.Fatalf("reconcileRabbitmq() error = %v", err)
	}
	updated := &backupsv1alpha1.BackupJob{}
	if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
		t.Fatalf("get backupjob: %v", err)
	}
	if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
		t.Errorf("expected phase Failed for wrong kind, got %q", updated.Status.Phase)
	}
}

// rabbitmqBackup builds a source Backup produced by the Rabbitmq strategy, for
// the restore-path tests.
func rabbitmqBackup(sourceApp string) *backupsv1alpha1.Backup {
	return &backupsv1alpha1.Backup{
		ObjectMeta: metav1.ObjectMeta{Name: "rmq-src-adhoc", Namespace: "tenant-test"},
		Spec: backupsv1alpha1.BackupSpec{
			ApplicationRef: newRabbitmqRef(sourceApp),
			StrategyRef: corev1.TypedLocalObjectReference{
				APIGroup: new(strategyv1alpha1.GroupVersion.Group),
				Kind:     strategyv1alpha1.RabbitmqStrategyKind,
				Name:     "cozy-default-rabbitmq",
			},
			TakenAt: metav1.Now(),
		},
		Status: backupsv1alpha1.BackupStatus{
			Artifact: &backupsv1alpha1.BackupArtifact{
				URI: "s3://bkt/tenant-test/" + sourceApp + "/rmq-src-adhoc/definitions.json",
			},
		},
	}
}

func rabbitmqRunningRestoreJob(name, targetApp string) *backupsv1alpha1.RestoreJob {
	now := metav1.Now()
	rj := &backupsv1alpha1.RestoreJob{
		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test"},
		Spec: backupsv1alpha1.RestoreJobSpec{
			BackupRef: corev1.LocalObjectReference{Name: "rmq-src-adhoc"},
		},
		Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
	}
	if targetApp != "" {
		rj.Spec.TargetApplicationRef = &corev1.TypedLocalObjectReference{
			APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
			Kind:     rabbitmqAppKind,
			Name:     targetApp,
		}
	}
	return rj
}

// TestReconcileRabbitmqRestore_ToCopy pins the to-copy path: the restore Job
// renders against the TARGET (targetApplicationRef override), not the source.
func TestReconcileRabbitmqRestore_ToCopy(t *testing.T) {
	targetApp := newRabbitmqApp("rmq-copy", "tenant-test")
	restoreJob := rabbitmqRunningRestoreJob("rmq-src-to-copy", "rmq-copy")
	backup := rabbitmqBackup("rmq-src")
	cluster := newRabbitmqCluster("rmq-copy", "tenant-test", true)

	_, rr := newRabbitmqTestEnv(t, targetApp, clientfake.NewClientBuilder().
		WithObjects(newRabbitmqStrategy("cozy-default-rabbitmq"), backup, restoreJob, cluster))
	ctx := context.Background()

	if _, err := rr.reconcileRabbitmqRestore(ctx, restoreJob, backup); err != nil {
		t.Fatalf("reconcileRabbitmqRestore() error = %v", err)
	}
	jobs := &batchv1.JobList{}
	if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	if len(jobs.Items) != 1 {
		t.Fatalf("expected 1 restore Job, got %d", len(jobs.Items))
	}
	if got := jobs.Items[0].Labels[rabbitmqLabelMode]; got != rabbitmqModeRestore {
		t.Errorf("mode label = %q, want %q", got, rabbitmqModeRestore)
	}
	args := jobs.Items[0].Spec.Template.Spec.Containers[0].Args
	// --host renders to the TARGET broker (rmq-copy), proving targetApplicationRef
	// override drove the render; --mode=restore; --uri is the URI RECORDED on the
	// source Backup (not a reconstruction), proving restore reads status.artifact.uri.
	want := []string{
		"--host=rabbitmq-rmq-copy.tenant-test.svc",
		"--mode=restore",
		"--uri=s3://bkt/tenant-test/rmq-src/rmq-src-adhoc/definitions.json",
	}
	for i, w := range want {
		if i >= len(args) || args[i] != w {
			t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
		}
	}
}

// TestReconcileRabbitmqRestore_TargetNotFound pins that a to-copy target that is
// not deployed fails the RestoreJob terminally rather than hanging.
func TestReconcileRabbitmqRestore_TargetNotFound(t *testing.T) {
	// The dynamic client only serves rmq-src, so the rmq-missing target Get
	// returns NotFound.
	seeded := newRabbitmqApp("rmq-src", "tenant-test")
	restoreJob := rabbitmqRunningRestoreJob("rmq-src-to-missing", "rmq-missing")
	backup := rabbitmqBackup("rmq-src")

	_, rr := newRabbitmqTestEnv(t, seeded, clientfake.NewClientBuilder().
		WithObjects(newRabbitmqStrategy("cozy-default-rabbitmq"), backup, restoreJob))
	ctx := context.Background()

	if _, err := rr.reconcileRabbitmqRestore(ctx, restoreJob, backup); err != nil {
		t.Fatalf("reconcileRabbitmqRestore() error = %v", err)
	}
	updated := &backupsv1alpha1.RestoreJob{}
	if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), updated); err != nil {
		t.Fatalf("get restorejob: %v", err)
	}
	if updated.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
		t.Errorf("expected phase Failed for missing target, got %q", updated.Status.Phase)
	}
	if !strings.Contains(updated.Status.Message, "target RabbitMQ application not found") {
		t.Errorf("expected 'target not found' message, got %q", updated.Status.Message)
	}
}

// TestReconcileRabbitmqRestore_TargetClusterNotReady pins that the readiness
// gate defers a restore until the TARGET broker is ready.
func TestReconcileRabbitmqRestore_TargetClusterNotReady(t *testing.T) {
	targetApp := newRabbitmqApp("rmq-copy", "tenant-test")
	restoreJob := rabbitmqRunningRestoreJob("rmq-src-to-copy", "rmq-copy")
	backup := rabbitmqBackup("rmq-src")
	cluster := newRabbitmqCluster("rmq-copy", "tenant-test", false)

	_, rr := newRabbitmqTestEnv(t, targetApp, clientfake.NewClientBuilder().
		WithObjects(newRabbitmqStrategy("cozy-default-rabbitmq"), backup, restoreJob, cluster))
	ctx := context.Background()

	res, err := rr.reconcileRabbitmqRestore(ctx, restoreJob, backup)
	if err != nil {
		t.Fatalf("reconcileRabbitmqRestore() error = %v", err)
	}
	if res.RequeueAfter == 0 {
		t.Error("expected a requeue while the target cluster is not ready")
	}
	jobs := &batchv1.JobList{}
	if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	if len(jobs.Items) != 0 {
		t.Fatalf("expected no restore Job while the target is not ready, got %d", len(jobs.Items))
	}
	updated := &backupsv1alpha1.RestoreJob{}
	if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), updated); err != nil {
		t.Fatalf("get restorejob: %v", err)
	}
	cond := meta.FindStatusCondition(updated.Status.Conditions, "Ready")
	if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "RabbitMQClusterNotReady" {
		t.Errorf("expected Ready=False/RabbitMQClusterNotReady, got %+v", cond)
	}
}

// TestReconcileRabbitmqRestore_NoRecordedArtifactURI pins that a Backup with no
// recorded status.artifact.uri fails the restore terminally, instead of
// reconstructing a key that may not point at a real object.
func TestReconcileRabbitmqRestore_NoRecordedArtifactURI(t *testing.T) {
	targetApp := newRabbitmqApp("rmq-src", "tenant-test")
	restoreJob := rabbitmqRunningRestoreJob("rmq-src-in-place", "")
	backup := rabbitmqBackup("rmq-src")
	backup.Status.Artifact = nil

	_, rr := newRabbitmqTestEnv(t, targetApp, clientfake.NewClientBuilder().
		WithObjects(newRabbitmqStrategy("cozy-default-rabbitmq"), backup, restoreJob))
	ctx := context.Background()

	if _, err := rr.reconcileRabbitmqRestore(ctx, restoreJob, backup); err != nil {
		t.Fatalf("reconcileRabbitmqRestore() error = %v", err)
	}
	updated := &backupsv1alpha1.RestoreJob{}
	if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), updated); err != nil {
		t.Fatalf("get restorejob: %v", err)
	}
	if updated.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
		t.Errorf("expected phase Failed for a Backup with no artifact URI, got %q", updated.Status.Phase)
	}
	if !strings.Contains(updated.Status.Message, "no recorded artifact URI") {
		t.Errorf("expected 'no recorded artifact URI' message, got %q", updated.Status.Message)
	}
}

// TestReconcileRabbitmq_RequiresArtifactURITemplate pins that a strategy without
// artifactURITemplate fails the backup: the Backup would have no recorded
// location for a restore to read.
func TestReconcileRabbitmq_RequiresArtifactURITemplate(t *testing.T) {
	app := newRabbitmqApp("rmq-test", "tenant-test")
	strategy := newRabbitmqStrategy("cozy-default-rabbitmq")
	strategy.Spec.ArtifactURITemplate = ""
	now := metav1.Now()
	backupJob := &backupsv1alpha1.BackupJob{
		ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
		Spec: backupsv1alpha1.BackupJobSpec{
			ApplicationRef:  newRabbitmqRef("rmq-test"),
			BackupClassName: "cozy-default",
		},
		Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
	}
	cluster := newRabbitmqCluster("rmq-test", "tenant-test", true)
	r, _ := newRabbitmqTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy, cluster))
	ctx := context.Background()

	if _, err := r.reconcileRabbitmq(ctx, backupJob, newRabbitmqResolved("cozy-default-rabbitmq", nil)); err != nil {
		t.Fatalf("reconcileRabbitmq() error = %v", err)
	}
	updated := &backupsv1alpha1.BackupJob{}
	if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
		t.Fatalf("get backupjob: %v", err)
	}
	if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
		t.Errorf("expected phase Failed for a strategy with no artifactURITemplate, got %q", updated.Status.Phase)
	}
	if !strings.Contains(updated.Status.Message, "artifactURITemplate") {
		t.Errorf("expected 'artifactURITemplate' message, got %q", updated.Status.Message)
	}
}

// TestCreateRabbitmqBackupArtifact_ArtifactURI pins that a rendered artifact URI
// is recorded on the Backup, and an empty URI leaves status.artifact unset.
func TestCreateRabbitmqBackupArtifact_ArtifactURI(t *testing.T) {
	app := newRabbitmqApp("rmq-test", "tenant-test")
	backupJob := &backupsv1alpha1.BackupJob{
		ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
		Spec:       backupsv1alpha1.BackupJobSpec{ApplicationRef: newRabbitmqRef("rmq-test"), BackupClassName: "cozy-default"},
	}
	r, _ := newRabbitmqTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob))
	ctx := context.Background()
	resolved := newRabbitmqResolved("cozy-default-rabbitmq", nil)

	const wantURI = "s3://bkt/tenant-test/rmq-test/definitions.json"
	if _, err := r.createRabbitmqBackupArtifact(ctx, backupJob, resolved, wantURI); err != nil {
		t.Fatalf("createRabbitmqBackupArtifact() with URI: %v", err)
	}
	// Re-Get, not the returned struct: the real Backup CRD has no status
	// subresource, so Create persists status.artifact - the re-Get is what
	// proves it, and would fail if a status subresource were ever added.
	created := &backupsv1alpha1.Backup{}
	if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: "test-bj"}, created); err != nil {
		t.Fatalf("get Backup: %v", err)
	}
	if created.Status.Artifact == nil || created.Status.Artifact.URI != wantURI {
		t.Errorf("expected persisted artifact URI %q, got %+v", wantURI, created.Status.Artifact)
	}

	backupJob.Name = "test-bj-2"
	if _, err := r.createRabbitmqBackupArtifact(ctx, backupJob, resolved, ""); err != nil {
		t.Fatalf("createRabbitmqBackupArtifact() without URI: %v", err)
	}
	noURI := &backupsv1alpha1.Backup{}
	if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: "test-bj-2"}, noURI); err != nil {
		t.Fatalf("get Backup (no URI): %v", err)
	}
	if noURI.Status.Artifact != nil {
		t.Errorf("expected no persisted artifact when URI is empty, got %+v", noURI.Status.Artifact)
	}
}

// TestCleanupRabbitmqBackup_SpawnsDeleteJob pins that deleting a Backup spawns
// an ownerless, self-cleaning Job that deletes the recorded object in cleanup
// mode - so a retention-pruned Plan does not leak objects.
func TestCleanupRabbitmqBackup_SpawnsDeleteJob(t *testing.T) {
	testScheme := runtime.NewScheme()
	_ = scheme.AddToScheme(testScheme)
	_ = backupsv1alpha1.AddToScheme(testScheme)
	_ = strategyv1alpha1.AddToScheme(testScheme)

	backup := rabbitmqBackup("rmq-src") // status.artifact.uri is set by the fixture
	strategy := newRabbitmqStrategy("cozy-default-rabbitmq")
	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}}
	c := clientfake.NewClientBuilder().WithScheme(testScheme).
		WithStatusSubresource(&batchv1.Job{}).
		WithObjects(ns, backup, strategy).Build()
	// Empty CredentialsConfig: ProjectBackupCredentials is a no-op when disabled,
	// so the spawn path runs without a source Secret in the fake.
	r := &BackupReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
	ctx := context.Background()

	// First pass: spawns the delete Job and requeues (waits) rather than
	// letting the Backup go.
	res, err := r.cleanupRabbitmqBackup(ctx, backup)
	if err != nil {
		t.Fatalf("cleanupRabbitmqBackup() error = %v", err)
	}
	if res.RequeueAfter == 0 {
		t.Error("expected a requeue while the delete Job runs (must not orphan the object)")
	}
	jobs := &batchv1.JobList{}
	if err := c.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	if len(jobs.Items) != 1 {
		t.Fatalf("expected 1 cleanup Job, got %d", len(jobs.Items))
	}
	job := jobs.Items[0]
	if job.Name != "rmq-src-adhoc-cleanup" {
		t.Errorf("job name = %q, want rmq-src-adhoc-cleanup", job.Name)
	}
	if len(job.OwnerReferences) != 0 {
		t.Errorf("cleanup Job must be ownerless so it survives Backup deletion, got %d ownerRefs", len(job.OwnerReferences))
	}
	if job.Labels[rabbitmqLabelMode] != rabbitmqModeCleanup {
		t.Errorf("mode label = %q, want %q", job.Labels[rabbitmqLabelMode], rabbitmqModeCleanup)
	}
	if job.Spec.TTLSecondsAfterFinished == nil || job.Spec.ActiveDeadlineSeconds == nil {
		t.Error("expected TTLSecondsAfterFinished + ActiveDeadlineSeconds on the cleanup Job")
	}
	foundMode := false
	for _, a := range job.Spec.Template.Spec.Containers[0].Args {
		if a == "--mode=cleanup" {
			foundMode = true
		}
	}
	if !foundMode {
		t.Errorf("expected the pod rendered in cleanup mode (--mode=cleanup), got args %v", job.Spec.Template.Spec.Containers[0].Args)
	}

	// Once the delete Job completes, cleanup finishes (zero Result) and reaps
	// the Job - so the Backup is only released after the object is gone.
	job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
	if err := c.Status().Update(ctx, &job); err != nil {
		t.Fatalf("update job status: %v", err)
	}
	res, err = r.cleanupRabbitmqBackup(ctx, backup)
	if err != nil {
		t.Fatalf("cleanupRabbitmqBackup() second call error = %v", err)
	}
	if res.RequeueAfter != 0 || res.Requeue {
		t.Errorf("expected cleanup to finish once the Job completed, got requeue %+v", res)
	}
	if err := c.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	if len(jobs.Items) != 0 {
		t.Errorf("expected the completed cleanup Job to be reaped, got %d", len(jobs.Items))
	}
}

// TestCleanupRabbitmqBackup_NoArtifactNoop pins that a Backup with no recorded
// artifact spawns nothing (there is no object to delete).
func TestCleanupRabbitmqBackup_NoArtifactNoop(t *testing.T) {
	testScheme := runtime.NewScheme()
	_ = scheme.AddToScheme(testScheme)
	_ = backupsv1alpha1.AddToScheme(testScheme)
	_ = strategyv1alpha1.AddToScheme(testScheme)

	backup := rabbitmqBackup("rmq-src")
	backup.Status.Artifact = nil
	c := clientfake.NewClientBuilder().WithScheme(testScheme).WithObjects(backup, newRabbitmqStrategy("cozy-default-rabbitmq")).Build()
	r := &BackupReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
	ctx := context.Background()

	res, err := r.cleanupRabbitmqBackup(ctx, backup)
	if err != nil {
		t.Fatalf("cleanupRabbitmqBackup() error = %v", err)
	}
	if res.RequeueAfter != 0 || res.Requeue {
		t.Errorf("expected no requeue when there is nothing to delete, got %+v", res)
	}
	jobs := &batchv1.JobList{}
	if err := c.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	if len(jobs.Items) != 0 {
		t.Fatalf("expected no cleanup Job when there is no artifact, got %d", len(jobs.Items))
	}
}

// TestReconcileRabbitmq_ReadinessDeadline pins that the AllReplicasReady wait is
// bounded: a broker that never becomes ready fails the BackupJob past the
// deadline instead of requeuing in phase=Running forever.
func TestReconcileRabbitmq_ReadinessDeadline(t *testing.T) {
	app := newRabbitmqApp("rmq-test", "tenant-test")
	strategy := newRabbitmqStrategy("cozy-default-rabbitmq")
	started := metav1.NewTime(time.Now().Add(-31 * time.Minute))
	backupJob := &backupsv1alpha1.BackupJob{
		ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
		Spec:       backupsv1alpha1.BackupJobSpec{ApplicationRef: newRabbitmqRef("rmq-test"), BackupClassName: "cozy-default"},
		Status:     backupsv1alpha1.BackupJobStatus{StartedAt: &started, Phase: backupsv1alpha1.BackupJobPhaseRunning},
	}
	cluster := newRabbitmqCluster("rmq-test", "tenant-test", false) // never ready
	r, _ := newRabbitmqTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy, cluster))
	ctx := context.Background()

	if _, err := r.reconcileRabbitmq(ctx, backupJob, newRabbitmqResolved("cozy-default-rabbitmq", nil)); err != nil {
		t.Fatalf("reconcileRabbitmq() error = %v", err)
	}
	updated := &backupsv1alpha1.BackupJob{}
	if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
		t.Fatalf("get backupjob: %v", err)
	}
	if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
		t.Errorf("expected phase Failed past the readiness deadline, got %q", updated.Status.Phase)
	}
	if !strings.Contains(updated.Status.Message, "did not become ready within") {
		t.Errorf("expected a deadline message, got %q", updated.Status.Message)
	}
}

// TestCleanupRabbitmqBackup_SkipAnnotation pins the escape hatch: an annotated
// Backup is released without spawning a delete Job, so an operator can remove a
// Backup stuck Terminating because object storage is unreachable.
func TestCleanupRabbitmqBackup_SkipAnnotation(t *testing.T) {
	testScheme := runtime.NewScheme()
	_ = scheme.AddToScheme(testScheme)
	_ = backupsv1alpha1.AddToScheme(testScheme)
	_ = strategyv1alpha1.AddToScheme(testScheme)

	backup := rabbitmqBackup("rmq-src")
	backup.Annotations = map[string]string{"backups.cozystack.io/skip-artifact-cleanup": "true"}
	c := clientfake.NewClientBuilder().WithScheme(testScheme).
		WithObjects(backup, newRabbitmqStrategy("cozy-default-rabbitmq")).Build()
	r := &BackupReconciler{Client: c, Scheme: testScheme, Recorder: record.NewFakeRecorder(10)}
	ctx := context.Background()

	res, err := r.cleanupRabbitmqBackup(ctx, backup)
	if err != nil {
		t.Fatalf("cleanupRabbitmqBackup() error = %v", err)
	}
	if res.RequeueAfter != 0 || res.Requeue {
		t.Errorf("expected the annotation to release the Backup (no requeue), got %+v", res)
	}
	jobs := &batchv1.JobList{}
	if err := c.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
		t.Fatalf("list jobs: %v", err)
	}
	if len(jobs.Items) != 0 {
		t.Fatalf("expected no delete Job when cleanup is skipped by annotation, got %d", len(jobs.Items))
	}
}