cozystack/internal/backupcontroller/default_objects_gate_test.go

package backupcontroller

import (
	"context"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus/testutil"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	dynamicfake "k8s.io/client-go/dynamic/fake"
	k8stesting "k8s.io/client-go/testing"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"

	backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)

// gateGVKs are the kinds the gate touches in these tests: the
// strategy.backups.cozystack.io CRs cozy-default routes to, the Velero BSL,
// and the HelmRelease it patches. The RESTMapper and the dynamic fake's
// resource registry are both derived from this one list so the plural the
// gate resolves and the plural the fake serves cannot drift apart.
var gateGVKs = []schema.GroupVersionKind{
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "CNPG"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "MariaDB"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"},
	{Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"},
	{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"},
}

func gateScheme() *runtime.Scheme {
	scheme := runtime.NewScheme()
	_ = corev1.AddToScheme(scheme)
	_ = backupsv1alpha1.AddToScheme(scheme)
	return scheme
}

func gateListKinds() map[schema.GroupVersionResource]string {
	out := map[schema.GroupVersionResource]string{}
	for _, gvk := range gateGVKs {
		gvr, _ := meta.UnsafeGuessKindToResource(gvk)
		out[gvr] = gvk.Kind + "List"
	}
	return out
}

// gateRESTMapper resolves group+kind (the only coordinates a
// BackupClass strategyRef carries) to a resource, which is what the gate
// does in production through the dynamic RESTMapper. The group versions must
// be passed as defaults: RESTMapping is called without a version.
func gateRESTMapper() meta.RESTMapper {
	groupVersions := map[schema.GroupVersion]struct{}{}
	for _, gvk := range gateGVKs {
		groupVersions[gvk.GroupVersion()] = struct{}{}
	}
	var gvs []schema.GroupVersion
	for gv := range groupVersions {
		gvs = append(gvs, gv)
	}
	m := meta.NewDefaultRESTMapper(gvs)
	for _, gvk := range gateGVKs {
		scope := meta.RESTScopeRoot
		if gvk.Kind == "BackupStorageLocation" || gvk.Kind == "HelmRelease" {
			scope = meta.RESTScopeNamespace
		}
		m.Add(gvk, scope)
	}
	return m
}

// cozyDefaultBackupClass mirrors the routes the chart's
// backupclass-default.yaml renders unconditionally: it is the manifest of
// what must exist, which is exactly why the gate reads it instead of
// hard-coding a list of Strategy names.
func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass {
	ref := func(kind, name string) backupsv1alpha1.BackupClassStrategy {
		return backupsv1alpha1.BackupClassStrategy{
			Application: backupsv1alpha1.ApplicationSelector{
				APIGroup: new("apps.cozystack.io"),
				Kind:     kind + "App",
			},
			StrategyRef: corev1.TypedLocalObjectReference{
				APIGroup: new(strategyAPIGroup),
				Kind:     kind,
				Name:     name,
			},
		}
	}
	return &backupsv1alpha1.BackupClass{
		ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"},
		Spec: backupsv1alpha1.BackupClassSpec{
			Strategies: []backupsv1alpha1.BackupClassStrategy{
				ref("CNPG", "cozy-default-cnpg"),
				ref("MariaDB", "cozy-default-mariadb"),
				ref("Etcd", "cozy-default-etcd"),
				ref("Altinity", "cozy-default-altinity"),
				ref("Velero", "cozy-default-velero-vminstance"),
				ref("Velero", "cozy-default-velero-vmdisk"),
			},
		},
	}
}

func strategyObject(kind, name string) *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion(strategyAPIGroup + "/v1alpha1")
	u.SetKind(kind)
	u.SetName(name)
	return u
}

func helmReleaseObject() *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion("helm.toolkit.fluxcd.io/v2")
	u.SetKind("HelmRelease")
	u.SetNamespace("cozy-backup-controller")
	u.SetName("backupstrategy-controller")
	return u
}

// credentialsHelmReleaseObject is the platform bucket's <bucket>-system
// release: the one that renders the user-credentials Secret the gate reads
// to resolve the bucket name, behind its own install-time lookup.
func credentialsHelmReleaseObject() *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion("helm.toolkit.fluxcd.io/v2")
	u.SetKind("HelmRelease")
	u.SetNamespace("tenant-root")
	u.SetName("bucket-cozy-backups-system")
	return u
}

func suspended(u *unstructured.Unstructured) *unstructured.Unstructured {
	_ = unstructured.SetNestedField(u.Object, true, "spec", "suspend")
	return u
}

func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) (*DefaultObjectsGate, *dynamicfake.FakeDynamicClient) {
	t.Helper()
	dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gateListKinds(), dynObjs...)
	g := &DefaultObjectsGate{
		Client: fake.NewClientBuilder().WithScheme(gateScheme()).WithObjects(ctrlObjs...).Build(),
		Config: BackupCredentialsConfig{
			SourceNamespace:  "tenant-root",
			SourceSecretName: "bucket-cozy-backups-system-credentials",
			TargetSecretName: "cozy-backups-creds",
		},
		BackupClassName: "cozy-default",
		HelmRelease: types.NamespacedName{
			Namespace: "cozy-backup-controller",
			Name:      "backupstrategy-controller",
		},
		CredentialsHelmRelease: types.NamespacedName{
			Namespace: "tenant-root",
			Name:      "bucket-cozy-backups-system",
		},
		VeleroNamespace:  "cozy-velero",
		MinForceInterval: 5 * time.Minute,
		now:              func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
	}
	g.Interface = dyn
	g.RESTMapper = gateRESTMapper()
	return g, dyn
}

func sourceSecret(bucket string) *corev1.Secret {
	data := map[string][]byte{
		"accessKey": []byte("AK"),
		"secretKey": []byte("SK"),
		"endpoint":  []byte("s3.example.com"),
	}
	if bucket != "" {
		data["bucketName"] = []byte(bucket)
	}
	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system-credentials"},
		Data:       data,
	}
}

func bslObject() *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion("velero.io/v1")
	u.SetKind("BackupStorageLocation")
	u.SetNamespace("cozy-velero")
	u.SetName(defaultBackupStorageLocationName)
	return u
}

func forceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string {
	t.Helper()
	return annotationsOf(t, dyn, "cozy-backup-controller", "backupstrategy-controller")
}

func credentialsForceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string {
	t.Helper()
	return annotationsOf(t, dyn, "tenant-root", "bucket-cozy-backups-system")
}

func annotationsOf(t *testing.T, dyn *dynamicfake.FakeDynamicClient, namespace, name string) map[string]string {
	t.Helper()
	hr, err := dyn.Resource(helmReleaseGVR).Namespace(namespace).
		Get(context.Background(), name, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("get HelmRelease %s/%s: %v", namespace, name, err)
	}
	return hr.GetAnnotations()
}

// TestCheckForcesWhenObjectsMissing is the bug this gate exists for: the
// bucket name resolves, so a Helm re-render WOULD produce the gated
// templates, but the objects are absent because the install-time lookup was
// empty and helm-controller never re-rendered. The gate must force a real
// Helm upgrade — BOTH annotations, because requestedAt alone is a no-op for
// an unchanged release.
func TestCheckForcesWhenObjectsMissing(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced {
		t.Fatal("expected a forced Helm upgrade when default objects are missing")
	}
	// 6 strategyRefs (Velero twice under different names) + the BSL.
	if len(missing) != 7 {
		t.Fatalf("missing = %v, want 7 entries", missing)
	}
	ann := forceAnnotations(t, dyn)
	forceAt, ok := ann["reconcile.fluxcd.io/forceAt"]
	if !ok || forceAt == "" {
		t.Errorf("reconcile.fluxcd.io/forceAt not set: %v", ann)
	}
	requestedAt, ok := ann["reconcile.fluxcd.io/requestedAt"]
	if !ok || requestedAt == "" {
		t.Errorf("reconcile.fluxcd.io/requestedAt not set: %v", ann)
	}
	if forceAt != requestedAt {
		t.Errorf("forceAt (%q) and requestedAt (%q) must carry the same stamp", forceAt, requestedAt)
	}
}

// TestCheckNoopWhenAllPresent pins the steady state: no forced upgrades
// once the objects exist, otherwise the gate would rewrite the annotation
// forever and re-run a Helm upgrade every MinForceInterval for the life of
// the cluster.
func TestCheckNoopWhenAllPresent(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
		bslObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if len(missing) != 0 || forced {
		t.Fatalf("missing = %v, forced = %v, want none", missing, forced)
	}
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("HelmRelease annotated in the steady state: %v", ann)
	}
}

// TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent is the second half
// of the bug. The Secret the gate reads to resolve the bucket name is itself
// rendered behind an install-time lookup, by the bucket's <bucket>-system
// release — so it is subject to the identical permanent skip, one release
// earlier. While it is absent nothing downstream can resolve: no projected
// credentials, no Strategy CRs, no Velero, and migration 50 has no snapshot
// target. Forcing THIS chart's release would be useless (its own lookups
// depend on that Secret); the bucket release is the one to force.
func TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced {
		t.Fatal("expected the bucket release to be forced while the credentials Secret is absent")
	}
	if len(missing) != 1 || missing[0] != "Secret/bucket-cozy-backups-system-credentials" {
		t.Fatalf("missing = %v, want the credentials Secret", missing)
	}
	ann := credentialsForceAnnotations(t, dyn)
	if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/requestedAt"] == "" {
		t.Errorf("bucket release not forced with both annotations: %v", ann)
	}
	// This chart's own release must be left alone: its templates gate on the
	// same unresolved bucket, so forcing it would burn a Helm upgrade that
	// re-runs the same empty lookup.
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("own HelmRelease forced before the bucket resolved: %v", ann)
	}
}

// TestCheckForcesCredentialsReleaseWhenBucketNameEmpty pins the same
// treatment one state later: the Secret exists but carries no bucket name (a
// partial render, or a hand-written Secret missing the key). Only a
// re-render of the producing release can complete it.
func TestCheckForcesCredentialsReleaseWhenBucketNameEmpty(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret(""), cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced || len(missing) != 1 {
		t.Fatalf("missing = %v, forced = %v; want the credentials Secret forced", missing, forced)
	}
	if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
		t.Errorf("bucket release not forced: %v", ann)
	}
}

// TestCheckReportsCredentialsSecretWithoutBucketRelease covers external S3
// (backupStorage.provisionBucket=false): the Secret is admin-managed and no
// release renders it, so there is nothing to force — but it must still be
// reported missing, because that is the state an operator has to alert on.
func TestCheckReportsCredentialsSecretWithoutBucketRelease(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)
	g.CredentialsHelmRelease = types.NamespacedName{}

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced a release on the external-S3 path, where none renders the Secret")
	}
	if len(missing) != 1 {
		t.Fatalf("missing = %v, want the credentials Secret reported", missing)
	}
	if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("bucket release annotated with no coordinates configured: %v", ann)
	}
}

// TestCheckThrottlesTheTwoReleasesIndependently pins that forcing the bucket
// release does not eat the objects release's throttle budget. The two happen
// in sequence on a real bootstrap — credentials first, then the objects the
// resolved bucket unblocks — so a shared timestamp would delay the second
// force by a full MinForceInterval for no reason.
func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) {
	base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)
	g.now = func() time.Time { return base }

	// No source Secret yet: the bucket release is forced.
	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("first Check: forced = %v, err = %v; want the bucket release forced", forced, err)
	}

	// A minute later the Secret lands. The objects are still missing, and
	// this force must go through despite being well inside MinForceInterval
	// of the previous one.
	g.now = func() time.Time { return base.Add(time.Minute) }
	if err := g.Client.Create(context.Background(), sourceSecret("bucket-1a2b")); err != nil {
		t.Fatalf("create source Secret: %v", err)
	}
	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("second Check: %v", err)
	}
	if !forced {
		t.Fatal("objects release throttled by the earlier credentials force")
	}
	if len(missing) != 7 {
		t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing)
	}
	if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
		t.Errorf("own HelmRelease not forced: %v", ann)
	}
}

// TestForceSkipsSuspendedHelmRelease pins the suspend guard. helm-controller
// ignores forceAt/requestedAt while spec.suspend is true (`cozyhr suspend` is
// a standard dev workflow), so patching anyway would re-stamp every
// MinForceInterval for the whole suspension and climb
// cozystack_backup_default_objects_force_reconciles_total — which the runbook
// reads as "the forced render is not producing the objects", the wrong
// diagnosis. The objects must still be reported missing.
func TestForceSkipsSuspendedHelmRelease(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		suspended(helmReleaseObject()),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced reported true for a suspended HelmRelease that ignores the annotations")
	}
	if len(missing) != 7 {
		t.Fatalf("missing = %v, want the objects still reported", missing)
	}
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("suspended HelmRelease annotated: %v", ann)
	}
}

// TestForceSkipsSuspendedCredentialsHelmRelease pins the same guard on the
// bucket release, which an operator is far more likely to have suspended:
// it lives in a tenant namespace and is not part of this chart.
func TestForceSkipsSuspendedCredentialsHelmRelease(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		suspended(credentialsHelmReleaseObject()),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced reported true for a suspended bucket HelmRelease")
	}
	if len(missing) != 1 {
		t.Fatalf("missing = %v, want the credentials Secret still reported", missing)
	}
	if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("suspended bucket HelmRelease annotated: %v", ann)
	}
}

// TestCheckTimeoutStaysBelowPeriod pins the bound on a single check. The ticker
// loop is sequential and the manager context is only cancelled at shutdown, so
// an unbounded check that hangs on a stalled API call stops every later check
// for the life of the pod.
func TestCheckTimeoutStaysBelowPeriod(t *testing.T) {
	for _, tc := range []struct {
		name   string
		period time.Duration
		want   time.Duration
	}{
		{"unset falls back to the cap", 0, 30 * time.Second},
		{"default period halves", time.Minute, 30 * time.Second},
		{"short period halves", 10 * time.Second, 5 * time.Second},
		{"long period is capped", time.Hour, 30 * time.Second},
	} {
		t.Run(tc.name, func(t *testing.T) {
			g := &DefaultObjectsGate{Period: tc.period}
			got := g.checkTimeout()
			if got != tc.want {
				t.Fatalf("checkTimeout() = %v, want %v", got, tc.want)
			}
			if tc.period > 0 && got >= tc.period {
				t.Fatalf("checkTimeout() = %v, must stay below Period %v", got, tc.period)
			}
		})
	}
}

// TestCheckThrottlesRepeatedForces pins the throttle. A condition a
// re-render cannot fix (missing CRD, unreachable bucket) must not turn into
// a hot loop of Helm upgrades against helm-controller.
func TestCheckThrottlesRepeatedForces(t *testing.T) {
	g, _ := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
	)
	base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	g.now = func() time.Time { return base }

	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("first Check: forced = %v, err = %v; want forced", forced, err)
	}

	// Well inside MinForceInterval: still missing, but no second upgrade.
	g.now = func() time.Time { return base.Add(time.Minute) }
	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("second Check: %v", err)
	}
	if forced {
		t.Error("second Check forced inside MinForceInterval")
	}
	if len(missing) == 0 {
		t.Error("second Check should still report the objects as missing")
	}

	// Past MinForceInterval: force again, because the objects are still gone.
	g.now = func() time.Time { return base.Add(6 * time.Minute) }
	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("third Check: forced = %v, err = %v; want forced", forced, err)
	}
}

// TestMissingObjectsIgnoresUnmappedKinds pins that a strategyRef whose CRD
// is not installed is not counted as missing. No Helm re-render can create
// such an object, so counting it would force upgrades forever.
func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) {
	bc := cozyDefaultBackupClass()
	bc.Spec.Strategies = append(bc.Spec.Strategies, backupsv1alpha1.BackupClassStrategy{
		Application: backupsv1alpha1.ApplicationSelector{Kind: "Nonexistent"},
		StrategyRef: corev1.TypedLocalObjectReference{
			APIGroup: new(strategyAPIGroup),
			Kind:     "NotInstalled",
			Name:     "cozy-default-notinstalled",
		},
	})
	g, _ := newGate(t, []client.Object{sourceSecret("b"), bc},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
		bslObject(),
	)

	missing, err := g.missingObjects(context.Background(), bc)
	if err != nil {
		t.Fatalf("missingObjects: %v", err)
	}
	if len(missing) != 0 {
		t.Fatalf("missing = %v, want none (unmapped kind must be ignored)", missing)
	}
}

// TestMissingObjectsSkipsVeleroWhenNamespaceEmpty covers
// velero.bslEnabled=false: the chart gates BOTH the BSL and the Velero
// Strategy CRs off the same flag, so neither is rendered and their absence
// is not a defect. The BackupClass still routes to the Velero strategies
// unconditionally, which is the trap: without the skip the gate would count
// the never-rendered Velero CRs as missing and force a Helm upgrade every
// MinForceInterval forever. The Velero objects are deliberately NOT
// pre-created here, because with the BSL disabled the chart never renders
// them on a real cluster.
func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) {
	bc := cozyDefaultBackupClass()
	g, _ := newGate(t, []client.Object{sourceSecret("b"), bc},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
	)
	g.VeleroNamespace = ""

	missing, err := g.missingObjects(context.Background(), bc)
	if err != nil {
		t.Fatalf("missingObjects: %v", err)
	}
	if len(missing) != 0 {
		t.Fatalf("missing = %v, want none when Velero is disabled", missing)
	}
}

// TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent covers Velero being removed
// after bootstrap while bslEnabled=true: the velero.io API is no longer
// served, so the BSL cannot be re-rendered and must not be counted as
// missing. Resolving through the RESTMapper turns that into a NoMatch skip;
// the previous hardcoded-GVR Get returned a NotFound that would have looped
// the gate forever.
func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) {
	bc := cozyDefaultBackupClass()
	g, _ := newGate(t, []client.Object{sourceSecret("b"), bc},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
	)
	// Rebuild the RESTMapper without BackupStorageLocation: the velero.io API
	// is not served on this cluster.
	gvset := map[schema.GroupVersion]struct{}{}
	var kept []schema.GroupVersionKind
	for _, gvk := range gateGVKs {
		if gvk.Kind == "BackupStorageLocation" {
			continue
		}
		gvset[gvk.GroupVersion()] = struct{}{}
		kept = append(kept, gvk)
	}
	var gvs []schema.GroupVersion
	for gv := range gvset {
		gvs = append(gvs, gv)
	}
	m := meta.NewDefaultRESTMapper(gvs)
	for _, gvk := range kept {
		scope := meta.RESTScopeRoot
		if gvk.Kind == "HelmRelease" {
			scope = meta.RESTScopeNamespace
		}
		m.Add(gvk, scope)
	}
	g.RESTMapper = m

	missing, err := g.missingObjects(context.Background(), bc)
	if err != nil {
		t.Fatalf("missingObjects: %v", err)
	}
	if len(missing) != 0 {
		t.Fatalf("missing = %v, want none when the Velero API is absent", missing)
	}
}

// TestCheckSurfacesPatchFailure pins that a failed force is reported rather
// than silently recorded as done — and that lastForce is NOT advanced, so
// the next tick retries instead of waiting out MinForceInterval.
func TestCheckSurfacesPatchFailure(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
	)
	dyn.PrependReactor("patch", "helmreleases", func(k8stesting.Action) (bool, runtime.Object, error) {
		return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"}, "backupstrategy-controller", nil)
	})

	if _, forced, err := g.Check(context.Background()); err == nil || forced {
		t.Fatalf("forced = %v, err = %v; want an error and forced=false", forced, err)
	}
	if !g.lastForce.IsZero() {
		t.Error("lastForce advanced despite the patch failing; the next tick would be throttled")
	}
}

// TestMissingGaugeCoversTheUnresolvedPath pins the alerting contract. The
// gauge used to be written only after the bucket resolved, so the state the
// gate exists to catch — no credentials Secret, therefore no strategies —
// reported 0 or nothing at all, and an alert on it never fired. Deleting the
// source Secret later (credential rotation) had the same effect: the gauge
// froze at its last value.
func TestMissingGaugeCoversTheUnresolvedPath(t *testing.T) {
	defaultObjectsMissing.Reset()
	g, _ := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)

	if _, _, err := g.Check(context.Background()); err != nil {
		t.Fatalf("Check: %v", err)
	}
	if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 1 {
		t.Fatalf("cozystack_backup_default_objects_missing = %v, want 1 while the credentials Secret is absent", got)
	}
}

// TestCheckErrorsCounterMarksTheGaugeStale pins the other half of that
// contract. On an API error the gauge deliberately keeps its last value
// rather than flapping the alert, which means the gauge alone cannot tell
// "healthy" from "not evaluated". The errors counter is what closes that
// gap, and the runbook alerts on both.
func TestCheckErrorsCounterMarksTheGaugeStale(t *testing.T) {
	defaultObjectsMissing.Reset()
	defaultObjectsCheckErrors.Reset()
	// No BackupClass: the check cannot enumerate what must exist.
	g, _ := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b")},
		helmReleaseObject(),
	)

	if _, _, err := g.Check(context.Background()); err == nil {
		t.Fatal("Check succeeded with no BackupClass, want an error")
	}
	if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 {
		t.Fatalf("cozystack_backup_default_objects_check_errors_total = %v, want 1", got)
	}
	if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 0 {
		t.Fatalf("gauge = %v, want it left untouched by a failed check", got)
	}
}

// TestStartDisabledWithoutHelmRelease pins the opt-out: without release
// coordinates the runnable must return immediately instead of ticking with
// a nil target.
func TestStartDisabledWithoutHelmRelease(t *testing.T) {
	g, _ := newGate(t, []client.Object{sourceSecret("b"), cozyDefaultBackupClass()}, helmReleaseObject())
	g.HelmRelease = types.NamespacedName{}

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	done := make(chan error, 1)
	go func() { done <- g.Start(ctx) }()
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("Start: %v", err)
		}
	case <-ctx.Done():
		t.Fatal("Start did not return immediately when disabled")
	}
}

// TestCheckToleratesAbsentHelmRelease covers a plain `helm install` (local
// development): there is no HelmRelease to force, which is not an error to
// report every tick.
func TestCheckToleratesAbsentHelmRelease(t *testing.T) {
	g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()})

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced reported true with no HelmRelease to patch")
	}
	if len(missing) == 0 {
		t.Error("objects should still be reported as missing")
	}
}