cozystack/internal/backupcontroller/default_objects_gate_test.go

package backupcontroller

import (
	"context"
	"errors"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus/testutil"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	dynamicfake "k8s.io/client-go/dynamic/fake"
	k8stesting "k8s.io/client-go/testing"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"

	backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)

// gateGVKs are the kinds the gate touches in these tests: the
// strategy.backups.cozystack.io CRs cozy-default routes to, the Velero BSL,
// and the HelmRelease it patches. The RESTMapper and the dynamic fake's
// resource registry are both derived from this one list so the plural the
// gate resolves and the plural the fake serves cannot drift apart.
var gateGVKs = []schema.GroupVersionKind{
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "CNPG"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "MariaDB"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"},
	{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"},
	{Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"},
	{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"},
}

func gateScheme() *runtime.Scheme {
	scheme := runtime.NewScheme()
	_ = corev1.AddToScheme(scheme)
	_ = backupsv1alpha1.AddToScheme(scheme)
	return scheme
}

func gateListKinds() map[schema.GroupVersionResource]string {
	out := map[schema.GroupVersionResource]string{}
	for _, gvk := range gateGVKs {
		gvr, _ := meta.UnsafeGuessKindToResource(gvk)
		out[gvr] = gvk.Kind + "List"
	}
	return out
}

// gateRESTMapper resolves group+kind (the only coordinates a
// BackupClass strategyRef carries) to a resource, which is what the gate
// does in production through the dynamic RESTMapper. The group versions must
// be passed as defaults: RESTMapping is called without a version.
func gateRESTMapper() meta.RESTMapper {
	groupVersions := map[schema.GroupVersion]struct{}{}
	for _, gvk := range gateGVKs {
		groupVersions[gvk.GroupVersion()] = struct{}{}
	}
	var gvs []schema.GroupVersion
	for gv := range groupVersions {
		gvs = append(gvs, gv)
	}
	m := meta.NewDefaultRESTMapper(gvs)
	for _, gvk := range gateGVKs {
		scope := meta.RESTScopeRoot
		if gvk.Kind == "BackupStorageLocation" || gvk.Kind == "HelmRelease" {
			scope = meta.RESTScopeNamespace
		}
		m.Add(gvk, scope)
	}
	return m
}

// cozyDefaultBackupClass mirrors the routes the chart's
// backupclass-default.yaml renders unconditionally: it is the manifest of
// what must exist, which is exactly why the gate reads it instead of
// hard-coding a list of Strategy names.
func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass {
	ref := func(kind, name string) backupsv1alpha1.BackupClassStrategy {
		return backupsv1alpha1.BackupClassStrategy{
			Application: backupsv1alpha1.ApplicationSelector{
				APIGroup: new("apps.cozystack.io"),
				Kind:     kind + "App",
			},
			StrategyRef: corev1.TypedLocalObjectReference{
				APIGroup: new(strategyAPIGroup),
				Kind:     kind,
				Name:     name,
			},
		}
	}
	return &backupsv1alpha1.BackupClass{
		ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"},
		Spec: backupsv1alpha1.BackupClassSpec{
			Strategies: []backupsv1alpha1.BackupClassStrategy{
				ref("CNPG", "cozy-default-cnpg"),
				ref("MariaDB", "cozy-default-mariadb"),
				ref("Etcd", "cozy-default-etcd"),
				ref("Altinity", "cozy-default-altinity"),
				ref("Velero", "cozy-default-velero-vminstance"),
				ref("Velero", "cozy-default-velero-vmdisk"),
			},
		},
	}
}

// renderedEndpoint is what the chart stamps on every default object for the
// endpoint sourceSecret carries: the steady state after a render that saw it.
const renderedEndpoint = "https://s3.example.com"

func strategyObject(kind, name string) *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion(strategyAPIGroup + "/v1alpha1")
	u.SetKind(kind)
	u.SetName(name)
	u.SetAnnotations(map[string]string{endpointAnnotation: renderedEndpoint})
	return u
}

func withEndpoint(u *unstructured.Unstructured, endpoint string) *unstructured.Unstructured {
	u.SetAnnotations(map[string]string{endpointAnnotation: endpoint})
	return u
}

// unannotated is an object rendered by a chart that predates the endpoint
// annotation.
func unannotated(u *unstructured.Unstructured) *unstructured.Unstructured {
	u.SetAnnotations(nil)
	return u
}

// allDefaultObjects is every object cozy-default routes to, plus the BSL, as
// the chart renders them for sourceSecret's endpoint.
func allDefaultObjects() []runtime.Object {
	return []runtime.Object{
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
		bslObject(),
	}
}

// cosiSecret is the Secret the COSI sidecar writes for the platform bucket's
// BucketAccess, which user-credentials.yaml renders the credentials Secret
// from. The BucketInfo document is one read off a live cluster, with the keys
// and the host replaced.
func cosiSecret(endpoint string) *corev1.Secret {
	bucketInfo := `{"metadata": {"name": "bc-b8e30337-1821-418f-a4dd-163e8016aaee", "creationTimestamp": null}, ` +
		`"spec": {"bucketName": "bucket-1a2b", "authenticationType": "KEY", ` +
		`"secretS3": {"endpoint": "` + endpoint + `", "region": "", "accessKeyID": "AK", "accessSecretKey": "SK"}, ` +
		`"secretAzure": null, "protocols": ["s3"]}}`
	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system"},
		Data:       map[string][]byte{"BucketInfo": []byte(bucketInfo)},
	}
}

func helmReleaseObject() *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion("helm.toolkit.fluxcd.io/v2")
	u.SetKind("HelmRelease")
	u.SetNamespace("cozy-backup-controller")
	u.SetName("backupstrategy-controller")
	return u
}

// credentialsHelmReleaseObject is the platform bucket's <bucket>-system
// release: the one that renders the user-credentials Secret the gate reads
// to resolve the bucket name, behind its own install-time lookup.
func credentialsHelmReleaseObject() *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion("helm.toolkit.fluxcd.io/v2")
	u.SetKind("HelmRelease")
	u.SetNamespace("tenant-root")
	u.SetName("bucket-cozy-backups-system")
	return u
}

func suspended(u *unstructured.Unstructured) *unstructured.Unstructured {
	_ = unstructured.SetNestedField(u.Object, true, "spec", "suspend")
	return u
}

func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) (*DefaultObjectsGate, *dynamicfake.FakeDynamicClient) {
	t.Helper()
	dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gateListKinds(), dynObjs...)
	g := &DefaultObjectsGate{
		Client: fake.NewClientBuilder().WithScheme(gateScheme()).WithObjects(ctrlObjs...).Build(),
		Config: BackupCredentialsConfig{
			SourceNamespace:  "tenant-root",
			SourceSecretName: "bucket-cozy-backups-system-credentials",
			TargetSecretName: "cozy-backups-creds",
		},
		BackupClassName: "cozy-default",
		HelmRelease: types.NamespacedName{
			Namespace: "cozy-backup-controller",
			Name:      "backupstrategy-controller",
		},
		CredentialsHelmRelease: types.NamespacedName{
			Namespace: "tenant-root",
			Name:      "bucket-cozy-backups-system",
		},
		CredentialsCOSISecretName: "bucket-cozy-backups-system",
		VeleroNamespace:           "cozy-velero",
		MinForceInterval:          5 * time.Minute,
		now:                       func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
	}
	g.Interface = dyn
	g.RESTMapper = gateRESTMapper()
	return g, dyn
}

func sourceSecret(bucket string) *corev1.Secret {
	data := map[string][]byte{
		"accessKey": []byte("AK"),
		"secretKey": []byte("SK"),
		"endpoint":  []byte("s3.example.com"),
	}
	if bucket != "" {
		data["bucketName"] = []byte(bucket)
	}
	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system-credentials"},
		Data:       data,
	}
}

func bslObject() *unstructured.Unstructured {
	u := &unstructured.Unstructured{}
	u.SetAPIVersion("velero.io/v1")
	u.SetKind("BackupStorageLocation")
	u.SetNamespace("cozy-velero")
	u.SetName(defaultBackupStorageLocationName)
	u.SetAnnotations(map[string]string{endpointAnnotation: renderedEndpoint})
	return u
}

func forceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string {
	t.Helper()
	return annotationsOf(t, dyn, "cozy-backup-controller", "backupstrategy-controller")
}

func credentialsForceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string {
	t.Helper()
	return annotationsOf(t, dyn, "tenant-root", "bucket-cozy-backups-system")
}

func annotationsOf(t *testing.T, dyn *dynamicfake.FakeDynamicClient, namespace, name string) map[string]string {
	t.Helper()
	hr, err := dyn.Resource(helmReleaseGVR).Namespace(namespace).
		Get(context.Background(), name, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("get HelmRelease %s/%s: %v", namespace, name, err)
	}
	return hr.GetAnnotations()
}

// TestCheckForcesWhenObjectsMissing is the bug this gate exists for: the
// bucket name resolves, so a Helm re-render WOULD produce the gated
// templates, but the objects are absent because the install-time lookup was
// empty and helm-controller never re-rendered. The gate must force a real
// Helm upgrade — BOTH annotations, because requestedAt alone is a no-op for
// an unchanged release.
func TestCheckForcesWhenObjectsMissing(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced {
		t.Fatal("expected a forced Helm upgrade when default objects are missing")
	}
	// 6 strategyRefs (Velero twice under different names) + the BSL.
	if len(missing) != 7 {
		t.Fatalf("missing = %v, want 7 entries", missing)
	}
	ann := forceAnnotations(t, dyn)
	forceAt, ok := ann["reconcile.fluxcd.io/forceAt"]
	if !ok || forceAt == "" {
		t.Errorf("reconcile.fluxcd.io/forceAt not set: %v", ann)
	}
	requestedAt, ok := ann["reconcile.fluxcd.io/requestedAt"]
	if !ok || requestedAt == "" {
		t.Errorf("reconcile.fluxcd.io/requestedAt not set: %v", ann)
	}
	if forceAt != requestedAt {
		t.Errorf("forceAt (%q) and requestedAt (%q) must carry the same stamp", forceAt, requestedAt)
	}
}

// TestCheckNoopWhenAllPresent pins the steady state: no forced upgrades
// once the objects exist, otherwise the gate would rewrite the annotation
// forever and re-run a Helm upgrade every MinForceInterval for the life of
// the cluster.
func TestCheckNoopWhenAllPresent(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.example.com"), cozyDefaultBackupClass()},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
		bslObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if len(missing) != 0 || forced {
		t.Fatalf("missing = %v, forced = %v, want none", missing, forced)
	}
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("HelmRelease annotated in the steady state: %v", ann)
	}
}

// TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent is the second half
// of the bug. The Secret the gate reads to resolve the bucket name is itself
// rendered behind an install-time lookup, by the bucket's <bucket>-system
// release — so it is subject to the identical permanent skip, one release
// earlier. While it is absent nothing downstream can resolve: no projected
// credentials, no Strategy CRs, no Velero, and migration 50 has no snapshot
// target. Forcing THIS chart's release would be useless (its own lookups
// depend on that Secret); the bucket release is the one to force.
func TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced {
		t.Fatal("expected the bucket release to be forced while the credentials Secret is absent")
	}
	if len(missing) != 1 || missing[0] != "Secret/bucket-cozy-backups-system-credentials" {
		t.Fatalf("missing = %v, want the credentials Secret", missing)
	}
	ann := credentialsForceAnnotations(t, dyn)
	if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/requestedAt"] == "" {
		t.Errorf("bucket release not forced with both annotations: %v", ann)
	}
	// This chart's own release must be left alone: its templates gate on the
	// same unresolved bucket, so forcing it would burn a Helm upgrade that
	// re-runs the same empty lookup.
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("own HelmRelease forced before the bucket resolved: %v", ann)
	}
}

// TestCheckForcesCredentialsReleaseWhenBucketNameEmpty pins the same
// treatment one state later: the Secret exists but carries no bucket name (a
// partial render, or a hand-written Secret missing the key). Only a
// re-render of the producing release can complete it.
func TestCheckForcesCredentialsReleaseWhenBucketNameEmpty(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret(""), cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced || len(missing) != 1 {
		t.Fatalf("missing = %v, forced = %v; want the credentials Secret forced", missing, forced)
	}
	if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
		t.Errorf("bucket release not forced: %v", ann)
	}
}

// TestCheckReportsCredentialsSecretWithoutBucketRelease covers external S3
// (backupStorage.provisionBucket=false): the Secret is admin-managed and no
// release renders it, so there is nothing to force — but it must still be
// reported missing, because that is the state an operator has to alert on.
func TestCheckReportsCredentialsSecretWithoutBucketRelease(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)
	g.CredentialsHelmRelease = types.NamespacedName{}

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced a release on the external-S3 path, where none renders the Secret")
	}
	if len(missing) != 1 {
		t.Fatalf("missing = %v, want the credentials Secret reported", missing)
	}
	if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("bucket release annotated with no coordinates configured: %v", ann)
	}
}

// TestCheckThrottlesTheTwoReleasesIndependently pins that forcing the bucket
// release does not eat the objects release's throttle budget. The two happen
// in sequence on a real bootstrap — credentials first, then the objects the
// resolved bucket unblocks — so a shared timestamp would delay the second
// force by a full MinForceInterval for no reason.
func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) {
	base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)
	g.now = func() time.Time { return base }

	// No source Secret yet: the bucket release is forced.
	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("first Check: forced = %v, err = %v; want the bucket release forced", forced, err)
	}

	// A minute later the Secret lands. The objects are still missing, and
	// this force must go through despite being well inside MinForceInterval
	// of the previous one.
	g.now = func() time.Time { return base.Add(time.Minute) }
	if err := g.Client.Create(context.Background(), sourceSecret("bucket-1a2b")); err != nil {
		t.Fatalf("create source Secret: %v", err)
	}
	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("second Check: %v", err)
	}
	if !forced {
		t.Fatal("objects release throttled by the earlier credentials force")
	}
	if len(missing) != 7 {
		t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing)
	}
	if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
		t.Errorf("own HelmRelease not forced: %v", ann)
	}
}

// TestForceSkipsSuspendedHelmRelease pins the suspend guard. helm-controller
// ignores forceAt/requestedAt while spec.suspend is true (`cozyhr suspend` is
// a standard dev workflow), so patching anyway would re-stamp every
// MinForceInterval for the whole suspension and climb
// cozystack_backup_default_objects_force_reconciles_total — which the runbook
// reads as "the forced render is not producing the objects", the wrong
// diagnosis. The objects must still be reported missing.
func TestForceSkipsSuspendedHelmRelease(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		suspended(helmReleaseObject()),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced reported true for a suspended HelmRelease that ignores the annotations")
	}
	if len(missing) != 7 {
		t.Fatalf("missing = %v, want the objects still reported", missing)
	}
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("suspended HelmRelease annotated: %v", ann)
	}
}

// TestForceSkipsSuspendedCredentialsHelmRelease pins the same guard on the
// bucket release, which an operator is far more likely to have suspended:
// it lives in a tenant namespace and is not part of this chart.
func TestForceSkipsSuspendedCredentialsHelmRelease(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		suspended(credentialsHelmReleaseObject()),
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced reported true for a suspended bucket HelmRelease")
	}
	if len(missing) != 1 {
		t.Fatalf("missing = %v, want the credentials Secret still reported", missing)
	}
	if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("suspended bucket HelmRelease annotated: %v", ann)
	}
}

// TestCheckTimeoutStaysBelowPeriod pins the bound on a single check. The ticker
// loop is sequential and the manager context is only cancelled at shutdown, so
// an unbounded check that hangs on a stalled API call stops every later check
// for the life of the pod.
func TestCheckTimeoutStaysBelowPeriod(t *testing.T) {
	for _, tc := range []struct {
		name   string
		period time.Duration
		want   time.Duration
	}{
		{"unset falls back to the cap", 0, 30 * time.Second},
		{"default period halves", time.Minute, 30 * time.Second},
		{"short period halves", 10 * time.Second, 5 * time.Second},
		{"long period is capped", time.Hour, 30 * time.Second},
	} {
		t.Run(tc.name, func(t *testing.T) {
			g := &DefaultObjectsGate{Period: tc.period}
			got := g.checkTimeout()
			if got != tc.want {
				t.Fatalf("checkTimeout() = %v, want %v", got, tc.want)
			}
			if tc.period > 0 && got >= tc.period {
				t.Fatalf("checkTimeout() = %v, must stay below Period %v", got, tc.period)
			}
		})
	}
}

// TestCheckThrottlesRepeatedForces pins the throttle. A condition a
// re-render cannot fix (missing CRD, unreachable bucket) must not turn into
// a hot loop of Helm upgrades against helm-controller.
func TestCheckThrottlesRepeatedForces(t *testing.T) {
	g, _ := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
	)
	base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	g.now = func() time.Time { return base }

	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("first Check: forced = %v, err = %v; want forced", forced, err)
	}

	// Well inside MinForceInterval: still missing, but no second upgrade.
	g.now = func() time.Time { return base.Add(time.Minute) }
	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("second Check: %v", err)
	}
	if forced {
		t.Error("second Check forced inside MinForceInterval")
	}
	if len(missing) == 0 {
		t.Error("second Check should still report the objects as missing")
	}

	// Past MinForceInterval: force again, because the objects are still gone.
	g.now = func() time.Time { return base.Add(6 * time.Minute) }
	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("third Check: forced = %v, err = %v; want forced", forced, err)
	}
}

// TestMissingObjectsIgnoresUnmappedKinds pins that a strategyRef whose CRD
// is not installed is not counted as missing. No Helm re-render can create
// such an object, so counting it would force upgrades forever.
func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) {
	bc := cozyDefaultBackupClass()
	bc.Spec.Strategies = append(bc.Spec.Strategies, backupsv1alpha1.BackupClassStrategy{
		Application: backupsv1alpha1.ApplicationSelector{Kind: "Nonexistent"},
		StrategyRef: corev1.TypedLocalObjectReference{
			APIGroup: new(strategyAPIGroup),
			Kind:     "NotInstalled",
			Name:     "cozy-default-notinstalled",
		},
	})
	g, _ := newGate(t, []client.Object{sourceSecret("b"), bc},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
		bslObject(),
	)

	state, err := g.inspectObjects(context.Background(), bc, "")
	if err != nil {
		t.Fatalf("inspectObjects: %v", err)
	}
	missing := state.missing
	if len(missing) != 0 {
		t.Fatalf("missing = %v, want none (unmapped kind must be ignored)", missing)
	}
}

// TestMissingObjectsSkipsVeleroWhenNamespaceEmpty covers
// velero.bslEnabled=false: the chart gates BOTH the BSL and the Velero
// Strategy CRs off the same flag, so neither is rendered and their absence
// is not a defect. The BackupClass still routes to the Velero strategies
// unconditionally, which is the trap: without the skip the gate would count
// the never-rendered Velero CRs as missing and force a Helm upgrade every
// MinForceInterval forever. The Velero objects are deliberately NOT
// pre-created here, because with the BSL disabled the chart never renders
// them on a real cluster.
func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) {
	bc := cozyDefaultBackupClass()
	g, _ := newGate(t, []client.Object{sourceSecret("b"), bc},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
	)
	g.VeleroNamespace = ""

	state, err := g.inspectObjects(context.Background(), bc, "")
	if err != nil {
		t.Fatalf("inspectObjects: %v", err)
	}
	missing := state.missing
	if len(missing) != 0 {
		t.Fatalf("missing = %v, want none when Velero is disabled", missing)
	}
}

// TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent covers Velero being removed
// after bootstrap while bslEnabled=true: the velero.io API is no longer
// served, so the BSL cannot be re-rendered and must not be counted as
// missing. Resolving through the RESTMapper turns that into a NoMatch skip;
// the previous hardcoded-GVR Get returned a NotFound that would have looped
// the gate forever.
func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) {
	bc := cozyDefaultBackupClass()
	g, _ := newGate(t, []client.Object{sourceSecret("b"), bc},
		helmReleaseObject(),
		strategyObject("CNPG", "cozy-default-cnpg"),
		strategyObject("MariaDB", "cozy-default-mariadb"),
		strategyObject("Etcd", "cozy-default-etcd"),
		strategyObject("Altinity", "cozy-default-altinity"),
		strategyObject("Velero", "cozy-default-velero-vminstance"),
		strategyObject("Velero", "cozy-default-velero-vmdisk"),
	)
	// Rebuild the RESTMapper without BackupStorageLocation: the velero.io API
	// is not served on this cluster.
	gvset := map[schema.GroupVersion]struct{}{}
	var kept []schema.GroupVersionKind
	for _, gvk := range gateGVKs {
		if gvk.Kind == "BackupStorageLocation" {
			continue
		}
		gvset[gvk.GroupVersion()] = struct{}{}
		kept = append(kept, gvk)
	}
	var gvs []schema.GroupVersion
	for gv := range gvset {
		gvs = append(gvs, gv)
	}
	m := meta.NewDefaultRESTMapper(gvs)
	for _, gvk := range kept {
		scope := meta.RESTScopeRoot
		if gvk.Kind == "HelmRelease" {
			scope = meta.RESTScopeNamespace
		}
		m.Add(gvk, scope)
	}
	g.RESTMapper = m

	state, err := g.inspectObjects(context.Background(), bc, "")
	if err != nil {
		t.Fatalf("inspectObjects: %v", err)
	}
	missing := state.missing
	if len(missing) != 0 {
		t.Fatalf("missing = %v, want none when the Velero API is absent", missing)
	}
}

// TestCheckSurfacesPatchFailure pins that a failed force is reported rather
// than silently recorded as done — and that lastForce is NOT advanced, so
// the next tick retries instead of waiting out MinForceInterval.
func TestCheckSurfacesPatchFailure(t *testing.T) {
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		helmReleaseObject(),
	)
	dyn.PrependReactor("patch", "helmreleases", func(k8stesting.Action) (bool, runtime.Object, error) {
		return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"}, "backupstrategy-controller", nil)
	})

	if _, forced, err := g.Check(context.Background()); err == nil || forced {
		t.Fatalf("forced = %v, err = %v; want an error and forced=false", forced, err)
	}
	if !g.lastForce.IsZero() {
		t.Error("lastForce advanced despite the patch failing; the next tick would be throttled")
	}
}

// TestMissingGaugeCoversTheUnresolvedPath pins the alerting contract. The
// gauge used to be written only after the bucket resolved, so the state the
// gate exists to catch — no credentials Secret, therefore no strategies —
// reported 0 or nothing at all, and an alert on it never fired. Deleting the
// source Secret later (credential rotation) had the same effect: the gauge
// froze at its last value.
func TestMissingGaugeCoversTheUnresolvedPath(t *testing.T) {
	defaultObjectsMissing.Reset()
	g, _ := newGate(t,
		[]client.Object{cozyDefaultBackupClass()},
		helmReleaseObject(),
		credentialsHelmReleaseObject(),
	)

	if _, _, err := g.Check(context.Background()); err != nil {
		t.Fatalf("Check: %v", err)
	}
	if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 1 {
		t.Fatalf("cozystack_backup_default_objects_missing = %v, want 1 while the credentials Secret is absent", got)
	}
}

// TestCheckErrorsCounterMarksTheGaugeStale pins the other half of that
// contract. On an API error the gauge deliberately keeps its last value
// rather than flapping the alert, which means the gauge alone cannot tell
// "healthy" from "not evaluated". The errors counter is what closes that
// gap, and the runbook alerts on both.
func TestCheckErrorsCounterMarksTheGaugeStale(t *testing.T) {
	defaultObjectsMissing.Reset()
	defaultObjectsCheckErrors.Reset()
	// No BackupClass: the check cannot enumerate what must exist.
	g, _ := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b")},
		helmReleaseObject(),
	)

	if _, _, err := g.Check(context.Background()); err == nil {
		t.Fatal("Check succeeded with no BackupClass, want an error")
	}
	if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 {
		t.Fatalf("cozystack_backup_default_objects_check_errors_total = %v, want 1", got)
	}
	if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 0 {
		t.Fatalf("gauge = %v, want it left untouched by a failed check", got)
	}
}

// TestStartDisabledWithoutHelmRelease pins the opt-out: without release
// coordinates the runnable must return immediately instead of ticking with
// a nil target.
func TestStartDisabledWithoutHelmRelease(t *testing.T) {
	g, _ := newGate(t, []client.Object{sourceSecret("b"), cozyDefaultBackupClass()}, helmReleaseObject())
	g.HelmRelease = types.NamespacedName{}

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	done := make(chan error, 1)
	go func() { done <- g.Start(ctx) }()
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("Start: %v", err)
		}
	case <-ctx.Done():
		t.Fatal("Start did not return immediately when disabled")
	}
}

// TestCheckToleratesAbsentHelmRelease covers a plain `helm install` (local
// development): there is no HelmRelease to force, which is not an error to
// report every tick.
func TestCheckToleratesAbsentHelmRelease(t *testing.T) {
	g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()})

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if forced {
		t.Error("forced reported true with no HelmRelease to patch")
	}
	if len(missing) == 0 {
		t.Error("objects should still be reported as missing")
	}
}

// TestCheckForcesCredentialsReleaseWhenEndpointStale covers the credentials
// Secret outliving the endpoint it was rendered from. The COSI sidecar writes
// BucketInfo once per BucketAccess, so a re-granted BucketAccess (after a
// publishing.host change) carries the new endpoint while the credentials
// Secret <bucket>-system rendered earlier keeps the old one: the Secret
// exists, so the missing-Secret path never fires, and every consumer keeps
// the dead host.
func TestCheckForcesCredentialsReleaseWhenEndpointStale(t *testing.T) {
	objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()},
		objs...,
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil {
		t.Fatalf("Check: %v", err)
	}
	if !forced {
		t.Fatal("expected the bucket release forced while its credentials Secret carries a stale endpoint")
	}
	if len(missing) != 1 || !strings.HasPrefix(missing[0], "Secret/bucket-cozy-backups-system-credentials") {
		t.Fatalf("missing = %v, want the credentials Secret", missing)
	}
	if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
		t.Errorf("bucket release not forced: %v", ann)
	}
	// The objects were rendered from the stale Secret and agree with it;
	// re-rendering them before the Secret is fixed would bake the old
	// endpoint in again.
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("own HelmRelease forced while the credentials Secret is stale: %v", ann)
	}
}

// TestCheckCredentialsEndpointComparison pins the comparison against what
// user-credentials.yaml writes for a given BucketInfo endpoint (the scheme
// stripped, nothing else), so a Secret the chart rendered from the current
// BucketInfo never counts as stale and the bucket release is not forced on
// every throttle window.
func TestCheckCredentialsEndpointComparison(t *testing.T) {
	for _, tc := range []struct {
		name      string
		cosi      string
		wantForce bool
	}{
		{"https scheme stripped", "https://s3.example.com", false},
		{"http scheme stripped", "http://s3.example.com", false},
		{"bare host", "s3.example.com", false},
		{"other host", "https://s3.new.example.com", true},
		{"other port", "https://s3.example.com:8443", true},
		// A forced render from a BucketInfo with no endpoint would write an
		// empty endpoint into the Secret every driver reads (or, with the key
		// absent, fail the bucket release's render), never a working one.
		{"no endpoint in BucketInfo", "", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
			g, dyn := newGate(t,
				[]client.Object{sourceSecret("bucket-1a2b"), cosiSecret(tc.cosi), cozyDefaultBackupClass()},
				objs...,
			)
			if _, _, err := g.Check(context.Background()); err != nil {
				t.Fatalf("Check: %v", err)
			}
			forced := credentialsForceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"] != ""
			if forced != tc.wantForce {
				t.Fatalf("bucket release forced = %v, want %v", forced, tc.wantForce)
			}
		})
	}
}

// TestCheckSkipsCredentialsComparisonWithoutCOSISecret covers the window in
// which the BucketAccess is being re-granted: the COSI Secret is gone and
// there is nothing to compare against, which is not evidence of staleness.
func TestCheckSkipsCredentialsComparisonWithoutCOSISecret(t *testing.T) {
	objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
		objs...,
	)

	missing, forced, err := g.Check(context.Background())
	if err != nil || forced || len(missing) != 0 {
		t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
	}
	if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("bucket release forced with no COSI Secret to compare against: %v", ann)
	}
}

// TestCheckSkipsCredentialsComparisonWhenUnconfigured pins both switches that
// turn the comparison off: no COSI Secret name, and no bucket release (external
// S3, where the Secret is admin-managed and no BucketInfo backs it).
func TestCheckSkipsCredentialsComparisonWhenUnconfigured(t *testing.T) {
	for _, tc := range []struct {
		name  string
		apply func(g *DefaultObjectsGate)
	}{
		{"no COSI Secret name", func(g *DefaultObjectsGate) { g.CredentialsCOSISecretName = "" }},
		{"no bucket release", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease = types.NamespacedName{} }},
		{"no bucket release name", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease.Name = "" }},
		{"no bucket release namespace", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease.Namespace = "" }},
	} {
		t.Run(tc.name, func(t *testing.T) {
			objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
			g, dyn := newGate(t,
				[]client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()},
				objs...,
			)
			tc.apply(g)
			// The real API server rejects a Get with an empty name; the fake
			// answers NotFound, which would hide a lookup of an unset name.
			g.Client = interceptor.NewClient(g.Client.(client.WithWatch), interceptor.Funcs{
				Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
					if key.Name == "" {
						return errors.New("resource name may not be empty")
					}
					return c.Get(ctx, key, obj, opts...)
				},
			})
			missing, forced, err := g.Check(context.Background())
			if err != nil || forced || len(missing) != 0 {
				t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
			}
			if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
				t.Errorf("bucket release forced with the comparison disabled: %v", ann)
			}
		})
	}
}

// TestCheckReportsMalformedCOSISecret pins that a COSI Secret the gate cannot
// read is an error the check counts, not a silent "not stale".
func TestCheckReportsMalformedCOSISecret(t *testing.T) {
	defaultObjectsCheckErrors.Reset()
	bad := cosiSecret("")
	bad.Data["BucketInfo"] = []byte("{not json")
	objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
	g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), bad, cozyDefaultBackupClass()}, objs...)

	if _, forced, err := g.Check(context.Background()); err == nil || forced {
		t.Fatalf("forced = %v, err = %v; want an error", forced, err)
	}
	if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 {
		t.Fatalf("check errors = %v, want 1", got)
	}
}

// TestCheckForcesOwnReleaseWhenObjectEndpointStale is the second half of a
// publishing.host change: the credentials Secret now carries the new
// endpoint, but the BSL and the default Strategies baked the old one at the
// last render of this chart and nothing re-renders them, so every backup
// keeps going to the dead host.
func TestCheckForcesOwnReleaseWhenObjectEndpointStale(t *testing.T) {
	for _, tc := range []struct {
		name  string
		stale *unstructured.Unstructured
		key   string
	}{
		{"strategy", strategyObject("CNPG", "cozy-default-cnpg"), "CNPG/cozy-default-cnpg"},
		{"BSL", bslObject(), "BackupStorageLocation/cozy-default"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var objs []runtime.Object
			for _, o := range allDefaultObjects() {
				u := o.(*unstructured.Unstructured)
				if u.GetKind() == tc.stale.GetKind() && u.GetName() == tc.stale.GetName() {
					withEndpoint(u, "https://s3.old.example.com")
				}
				objs = append(objs, u)
			}
			objs = append(objs, helmReleaseObject(), credentialsHelmReleaseObject())
			g, dyn := newGate(t,
				[]client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.example.com"), cozyDefaultBackupClass()},
				objs...,
			)

			missing, forced, err := g.Check(context.Background())
			if err != nil {
				t.Fatalf("Check: %v", err)
			}
			if !forced {
				t.Fatal("expected this chart's release forced while an object carries a stale endpoint")
			}
			if len(missing) != 1 || !strings.HasPrefix(missing[0], tc.key+" ") {
				t.Fatalf("missing = %v, want only %s", missing, tc.key)
			}
			if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
				t.Errorf("own HelmRelease not forced: %v", ann)
			}
			if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
				t.Errorf("bucket release forced for a stale object: %v", ann)
			}
		})
	}
}

// TestCheckIgnoresObjectEndpointOnExternalS3 pins that the comparison is off
// when no bucket release is configured: with provisionBucket=false the chart
// renders backupStorage.endpoint, not the Secret's, so the two legitimately
// differ and forcing would loop every MinForceInterval.
func TestCheckIgnoresObjectEndpointOnExternalS3(t *testing.T) {
	objs := []runtime.Object{helmReleaseObject()}
	for _, o := range allDefaultObjects() {
		objs = append(objs, withEndpoint(o.(*unstructured.Unstructured), "https://minio.example.internal:9000"))
	}
	g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...)
	g.CredentialsHelmRelease = types.NamespacedName{}

	missing, forced, err := g.Check(context.Background())
	if err != nil || forced || len(missing) != 0 {
		t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
	}
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("own HelmRelease forced on external S3: %v", ann)
	}
}

// TestCheckIgnoresObjectEndpointWithoutSecretEndpoint covers a credentials
// Secret with no endpoint key: the chart then renders its values fallback,
// which the gate cannot reproduce, so there is nothing to compare against.
func TestCheckIgnoresObjectEndpointWithoutSecretEndpoint(t *testing.T) {
	src := sourceSecret("bucket-1a2b")
	delete(src.Data, "endpoint")
	objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
	g, dyn := newGate(t, []client.Object{src, cozyDefaultBackupClass()}, objs...)

	if _, forced, err := g.Check(context.Background()); err != nil || forced {
		t.Fatalf("forced = %v, err = %v; want a no-op", forced, err)
	}
	if ann := forceAnnotations(t, dyn); len(ann) != 0 {
		t.Errorf("own HelmRelease forced with no endpoint to compare against: %v", ann)
	}
}

// TestCheckKeepsForcingForUnannotatedObjects covers objects rendered by a
// chart that predates the endpoint annotation. Their endpoint is unknown, so
// they count as stale until a render stamps the annotation. Having requested
// a render proves nothing: one that failed leaves them unannotated, and the
// next unthrottled check must force again rather than fall silent, so the
// force counter keeps reporting the render that does not land.
func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) {
	base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	objs := []runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}
	for _, o := range allDefaultObjects() {
		objs = append(objs, unannotated(o.(*unstructured.Unstructured)))
	}
	g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...)
	g.lastForce = base.Add(-time.Minute)
	g.now = func() time.Time { return base }

	if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 {
		t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err)
	}

	g.now = func() time.Time { return base.Add(5 * time.Minute) }
	if _, forced, err := g.Check(context.Background()); err != nil || !forced {
		t.Fatalf("first unthrottled Check: forced = %v, err = %v; want forced", forced, err)
	}
	first := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]

	g.now = func() time.Time { return base.Add(time.Hour) }
	missing, forced, err := g.Check(context.Background())
	if err != nil || !forced || len(missing) != 7 {
		t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err)
	}
	if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first {
		t.Errorf("not forced again for unannotated objects: forceAt still %q", first)
	}
}

// TestRenderedEndpointMatchesTheChart pins renderedEndpointFor to the
// "backupstrategy-controller.endpoint" helper, which the chart stamps as the
// annotation: trim, strip a scheme, force https://. Any divergence makes the
// gate see every object as stale and force this release forever.
func TestRenderedEndpointMatchesTheChart(t *testing.T) {
	for _, tc := range []struct{ in, want string }{
		{"s3.example.com", "https://s3.example.com"},
		{"s3.example.com\n", "https://s3.example.com"},
		{" s3.example.com ", "https://s3.example.com"},
		{"https://s3.example.com", "https://s3.example.com"},
		{"http://s3.example.com:8333", "https://s3.example.com:8333"},
		{"s3.example.com/prefix/", "https://s3.example.com/prefix/"},
		// No endpoint in the Secret: the chart renders its values fallback,
		// which the gate cannot reproduce, so it compares nothing.
		{"", ""},
		// Present but blank: the helper's lookup sees a non-empty value and
		// renders the bare scheme.
		{"\n", "https://"},
	} {
		if got := renderedEndpointFor([]byte(tc.in)); got != tc.want {
			t.Errorf("renderedEndpointFor(%q) = %q, want %q", tc.in, got, tc.want)
		}
	}
}

// TestForcedReleasesForNamesEachRenderingRelease pins the releases the log
// line names: the credentials Secret, missing or stale, is repaired by the
// bucket release, everything else by this chart's own, and both are named when
// one check forced both.
func TestForcedReleasesForNamesEachRenderingRelease(t *testing.T) {
	g, _ := newGate(t, nil)
	creds, own := g.CredentialsHelmRelease.String(), g.HelmRelease.String()
	for _, tc := range []struct {
		missing []string
		want    []string
	}{
		{[]string{"Secret/bucket-cozy-backups-system-credentials"}, []string{creds}},
		{[]string{`Secret/bucket-cozy-backups-system-credentials (endpoint "s3.old", COSI Secret has "s3.new")`}, []string{creds}},
		{[]string{"Secret/bucket-cozy-backups-system-credentials-other"}, []string{own}},
		{[]string{`CNPG/cozy-default-cnpg (endpoint "https://s3.old", want "https://s3.new")`, "Etcd/cozy-default-etcd"}, []string{own}},
		{[]string{`Secret/bucket-cozy-backups-system-credentials (endpoint "s3.old", COSI Secret has "s3.new")`, "CNPG/cozy-default-cnpg"}, []string{creds, own}},
	} {
		if got := g.forcedReleasesFor(tc.missing); !slices.Equal(got, tc.want) {
			t.Errorf("forcedReleasesFor(%v) = %v, want %v", tc.missing, got, tc.want)
		}
	}
	// External S3: no bucket release, so nothing but this chart's own.
	g.CredentialsHelmRelease = types.NamespacedName{}
	if got := g.forcedReleasesFor([]string{"Secret/bucket-cozy-backups-system-credentials"}); !slices.Equal(got, []string{own}) {
		t.Errorf("forcedReleasesFor without a bucket release = %v, want [%s]", got, own)
	}
}

// TestCheckSkipsCredentialsComparisonWithoutDerivedSecret covers a
// systemSecretName the bucket chart does not render: the same name as the COSI
// Secret (the projector also reads a raw COSI Secret) or a Secret without the
// endpoint key. Comparing it would report it stale on every tick, or fail the
// check, and the missing objects would never be repaired.
func TestCheckSkipsCredentialsComparisonWithoutDerivedSecret(t *testing.T) {
	for _, tc := range []struct {
		name  string
		apply func(g *DefaultObjectsGate, src *corev1.Secret)
	}{
		{"COSI Secret is the source", func(g *DefaultObjectsGate, src *corev1.Secret) {
			g.CredentialsCOSISecretName = g.Config.SourceSecretName
			src.Data["BucketInfo"] = cosiSecret("https://s3.new.example.com").Data["BucketInfo"]
		}},
		{"source without endpoint", func(_ *DefaultObjectsGate, src *corev1.Secret) {
			delete(src.Data, "endpoint")
		}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			src := sourceSecret("bucket-1a2b")
			var objs []runtime.Object
			for _, o := range allDefaultObjects() {
				if u := o.(*unstructured.Unstructured); u.GetName() != "cozy-default-cnpg" {
					objs = append(objs, u)
				}
			}
			objs = append(objs, helmReleaseObject(), credentialsHelmReleaseObject())
			g, dyn := newGate(t, nil, objs...)
			tc.apply(g, src)
			g.Client = fake.NewClientBuilder().WithScheme(gateScheme()).
				WithObjects(src, cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()).Build()

			missing, forced, err := g.Check(context.Background())
			if err != nil {
				t.Fatalf("Check: %v", err)
			}
			if !forced || len(missing) != 1 || missing[0] != "CNPG/cozy-default-cnpg" {
				t.Fatalf("missing = %v, forced = %v; want the missing strategy repaired", missing, forced)
			}
			if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
				t.Errorf("bucket release forced: %v", ann)
			}
		})
	}
}

// TestCheckRepairsMissingObjectsWhileCredentialsUnresolved pins that a stale
// or unreadable credentials comparison does not cost the repair of missing
// objects the gate made before it existed. The comparison only withholds the
// endpoint check, whose answer would be rendered from the same stale Secret.
func TestCheckRepairsMissingObjectsWhileCredentialsUnresolved(t *testing.T) {
	malformed := cosiSecret("")
	malformed.Data["BucketInfo"] = []byte("{not json")
	for _, tc := range []struct {
		name          string
		cosi          *corev1.Secret
		credsHR       *unstructured.Unstructured
		wantErr       bool
		wantCredsHRAt bool
	}{
		{"stale credentials", cosiSecret("https://s3.new.example.com"), credentialsHelmReleaseObject(), false, true},
		{"stale credentials, bucket release suspended", cosiSecret("https://s3.new.example.com"), suspended(credentialsHelmReleaseObject()), false, false},
		{"malformed COSI Secret", malformed, credentialsHelmReleaseObject(), true, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			defaultObjectsCheckErrors.Reset()
			defaultObjectsMissing.Reset()
			var objs []runtime.Object
			for _, o := range allDefaultObjects() {
				if u := o.(*unstructured.Unstructured); u.GetName() != "cozy-default-cnpg" {
					objs = append(objs, u)
				}
			}
			objs = append(objs, helmReleaseObject(), tc.credsHR)
			g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), tc.cosi, cozyDefaultBackupClass()}, objs...)

			missing, forced, err := g.Check(context.Background())
			if (err != nil) != tc.wantErr {
				t.Fatalf("err = %v, want error %v", err, tc.wantErr)
			}
			if !forced {
				t.Error("forced = false, want this chart's release forced for the missing strategy")
			}
			if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
				t.Errorf("own HelmRelease not forced for a missing strategy: %v", ann)
			}
			if got := credentialsForceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"] != ""; got != tc.wantCredsHRAt {
				t.Errorf("bucket release forced = %v, want %v", got, tc.wantCredsHRAt)
			}
			found := false
			for _, m := range missing {
				found = found || m == "CNPG/cozy-default-cnpg"
			}
			if !found {
				t.Errorf("missing = %v, want CNPG/cozy-default-cnpg reported", missing)
			}
			if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 1 {
				t.Errorf("missing gauge = %v, want 1", got)
			}
			wantErrs := 0.0
			if tc.wantErr {
				wantErrs = 1
			}
			if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != wantErrs {
				t.Errorf("check errors = %v, want %v", got, wantErrs)
			}
		})
	}
}

// TestCheckCountsOneErrorPerCheck pins that a check failing in two places is
// still one check that did not reach a conclusion.
func TestCheckCountsOneErrorPerCheck(t *testing.T) {
	defaultObjectsCheckErrors.Reset()
	bad := cosiSecret("")
	bad.Data["BucketInfo"] = []byte("{not json")
	g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), bad}, helmReleaseObject(), credentialsHelmReleaseObject())

	if _, _, err := g.Check(context.Background()); err == nil {
		t.Fatal("Check succeeded with a malformed COSI Secret and no BackupClass")
	}
	if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 {
		t.Fatalf("check errors = %v, want 1", got)
	}
}

// TestCheckSurfacesCredentialsPatchFailure pins that a failed force of the
// bucket release is reported, not recorded as done.
func TestCheckSurfacesCredentialsPatchFailure(t *testing.T) {
	objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
	g, dyn := newGate(t,
		[]client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()},
		objs...,
	)
	dyn.PrependReactor("patch", "helmreleases", func(k8stesting.Action) (bool, runtime.Object, error) {
		return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"}, "bucket-cozy-backups-system", nil)
	})

	if _, forced, err := g.Check(context.Background()); err == nil || forced {
		t.Fatalf("forced = %v, err = %v; want an error and forced=false", forced, err)
	}
	if !g.lastCredentialsForce.IsZero() {
		t.Error("lastCredentialsForce advanced despite the patch failing")
	}
}

// TestCheckForcesStaleCredentialsWhenObjectsUnreadable pins that the
// credentials comparison needs only the two Secrets: a BackupClass or a routed
// object the gate cannot read must not keep the stale endpoint in the
// credentials Secret, which every other driver reads through the projector.
func TestCheckForcesStaleCredentialsWhenObjectsUnreadable(t *testing.T) {
	for _, tc := range []struct {
		name        string
		backupClass bool
		forbidCNPG  bool
	}{
		{"BackupClass absent", false, false},
		{"routed Strategy unreadable", true, true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			defaultObjectsCheckErrors.Reset()
			ctrlObjs := []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com")}
			if tc.backupClass {
				ctrlObjs = append(ctrlObjs, cozyDefaultBackupClass())
			}
			objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
			g, dyn := newGate(t, ctrlObjs, objs...)
			if tc.forbidCNPG {
				dyn.PrependReactor("get", "cnpgs", func(k8stesting.Action) (bool, runtime.Object, error) {
					return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: strategyAPIGroup, Resource: "cnpgs"}, "cozy-default-cnpg", nil)
				})
			}

			missing, forced, err := g.Check(context.Background())
			if err == nil {
				t.Fatal("Check succeeded although the routed objects could not be evaluated")
			}
			if !forced {
				t.Error("forced = false, want the bucket release forced for its stale credentials Secret")
			}
			if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
				t.Errorf("bucket release not forced: %v", ann)
			}
			if ann := forceAnnotations(t, dyn); len(ann) != 0 {
				t.Errorf("own HelmRelease forced without a conclusive object check: %v", ann)
			}
			if len(missing) != 1 || !strings.HasPrefix(missing[0], "Secret/bucket-cozy-backups-system-credentials") {
				t.Errorf("missing = %v, want the credentials Secret", missing)
			}
			if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 {
				t.Errorf("check errors = %v, want 1", got)
			}
		})
	}
}