mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-05 03:27:07 +09:00
338 lines
13 KiB
Go
338 lines
13 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
|
)
|
|
|
|
// TestStrategyKindForRestoreJob asserts the dispatcher correctly identifies
|
|
// the strategy that produced a RestoreJob's side state. Locks in the bug
|
|
// fix: previously the cleanup ran Velero-only deletion regardless of the
|
|
// RestoreJob's actual strategy, papered over by the fact that DeleteAllOf
|
|
// is a no-op when nothing matches the label selector.
|
|
func TestStrategyKindForRestoreJob(t *testing.T) {
|
|
apiGroup := strategyv1alpha1.GroupVersion.Group
|
|
cases := []struct {
|
|
name string
|
|
backupKind string
|
|
want string
|
|
}{
|
|
{"velero strategy", strategyv1alpha1.VeleroStrategyKind, strategyv1alpha1.VeleroStrategyKind},
|
|
{"cnpg strategy", strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.CNPGStrategyKind},
|
|
{"job strategy", strategyv1alpha1.JobStrategyKind, strategyv1alpha1.JobStrategyKind},
|
|
{"mariadb strategy", strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MariaDBStrategyKind},
|
|
{"etcd strategy", strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.EtcdStrategyKind},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: "bk"},
|
|
},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &apiGroup,
|
|
Kind: tc.backupKind,
|
|
Name: "strategy",
|
|
},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj, backup)
|
|
got := strategyKindForRestoreJob(context.Background(), c, rj)
|
|
if got != tc.want {
|
|
t.Errorf("got %q want %q", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("missing backup yields empty kind", func(t *testing.T) {
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: "missing"},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj)
|
|
if got := strategyKindForRestoreJob(context.Background(), c, rj); got != "" {
|
|
t.Errorf("expected empty string for missing backup, got %q", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestCleanupOnDelete_CNPG_DoesNotTouchVeleroNamespace asserts the dispatch
|
|
// branch: a CNPG RestoreJob's deletion must not issue DeleteAllOf against
|
|
// cozy-velero. We seed a Velero Restore in cozy-velero that *would* match
|
|
// the CNPG RestoreJob's labels - so if the dispatcher mistakenly fell into
|
|
// the Velero cleanup branch, DeleteAllOf would reap it. The previous
|
|
// regression test seeded a different-owner stray, which left it alone
|
|
// regardless of which cleanup branch ran (DeleteAllOf with a label selector
|
|
// is a no-op against non-matching objects), so it passed even on the broken
|
|
// implementation.
|
|
func TestCleanupOnDelete_CNPG_DoesNotTouchVeleroNamespace(t *testing.T) {
|
|
apiGroup := strategyv1alpha1.GroupVersion.Group
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &apiGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "s",
|
|
},
|
|
},
|
|
}
|
|
// Pre-seed a Velero Restore that *matches* the CNPG RestoreJob's owner
|
|
// labels. CNPG dispatch must not enter cleanupVeleroRestore, so this
|
|
// object must survive. If the dispatcher regresses to the old
|
|
// unconditional Velero cleanup, DeleteAllOf will reap it and the test
|
|
// will fail.
|
|
owned := &velerov1.Restore{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: veleroNamespace,
|
|
Name: "would-be-victim",
|
|
Labels: map[string]string{
|
|
backupsv1alpha1.OwningJobNameLabel: "rj",
|
|
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
|
},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj, backup, owned)
|
|
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
r.cleanupOnDelete(context.Background(), rj)
|
|
|
|
got := &velerov1.Restore{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(owned), got); err != nil {
|
|
t.Fatalf("CNPG cleanup incorrectly routed to Velero cleanup; matching Velero Restore was deleted (err=%v)", err)
|
|
}
|
|
}
|
|
|
|
// TestCleanupOnDelete_FoundationDB_DoesNotTouchVeleroNamespace mirrors the
|
|
// CNPG cleanup test for the FoundationDB strategy. The dispatch switch in
|
|
// cleanupOnDelete groups CNPG / Job / Altinity / MariaDB / FoundationDB
|
|
// into the no-op cleanup branch (none of them materialise namespaced
|
|
// artefacts in cozy-velero that outlive the RestoreJob). A future
|
|
// refactor that drops FoundationDB from the no-op group would fall
|
|
// through to the conservative `default` branch which runs the Velero
|
|
// cleanup unconditionally - and silently start reaping matching
|
|
// velero.io/Restore objects in cozy-velero. Seeding a label-matching
|
|
// Velero Restore proves the FoundationDB branch stays no-op.
|
|
func TestCleanupOnDelete_FoundationDB_DoesNotTouchVeleroNamespace(t *testing.T) {
|
|
apiGroup := strategyv1alpha1.GroupVersion.Group
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &apiGroup, Kind: strategyv1alpha1.FoundationDBStrategyKind, Name: "s",
|
|
},
|
|
},
|
|
}
|
|
owned := &velerov1.Restore{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: veleroNamespace,
|
|
Name: "would-be-victim",
|
|
Labels: map[string]string{
|
|
backupsv1alpha1.OwningJobNameLabel: "rj",
|
|
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
|
},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj, backup, owned)
|
|
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
r.cleanupOnDelete(context.Background(), rj)
|
|
|
|
got := &velerov1.Restore{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(owned), got); err != nil {
|
|
t.Fatalf("FoundationDB cleanup incorrectly routed to Velero cleanup; matching Velero Restore was deleted (err=%v)", err)
|
|
}
|
|
}
|
|
|
|
// TestCleanupOnDelete_Etcd_DoesNotTouchVeleroNamespace mirrors the
|
|
// CNPG / FoundationDB cleanup tests for the Etcd strategy. The Etcd
|
|
// driver does not materialise namespaced artefacts in cozy-velero
|
|
// (the destructive in-place flow's side state lives on the RestoreJob
|
|
// itself as the EtcdClusterSpecCaptured + TargetPurged conditions,
|
|
// and the operator-side EtcdCluster is owned by the source
|
|
// HelmRelease). A future refactor that drops Etcd from the no-op
|
|
// group would fall through to the conservative `default` branch
|
|
// which runs the Velero cleanup unconditionally - silently reaping
|
|
// matching velero.io/Restore objects on label match and emitting a
|
|
// misleading "Failed to delete Velero Restore" event on
|
|
// non-Etcd-related apiserver errors. Seeding a label-matching Velero
|
|
// Restore proves the Etcd branch stays no-op.
|
|
func TestCleanupOnDelete_Etcd_DoesNotTouchVeleroNamespace(t *testing.T) {
|
|
apiGroup := strategyv1alpha1.GroupVersion.Group
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &apiGroup, Kind: strategyv1alpha1.EtcdStrategyKind, Name: "s",
|
|
},
|
|
},
|
|
}
|
|
owned := &velerov1.Restore{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: veleroNamespace,
|
|
Name: "would-be-victim",
|
|
Labels: map[string]string{
|
|
backupsv1alpha1.OwningJobNameLabel: "rj",
|
|
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
|
},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj, backup, owned)
|
|
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
r.cleanupOnDelete(context.Background(), rj)
|
|
|
|
got := &velerov1.Restore{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(owned), got); err != nil {
|
|
t.Fatalf("Etcd cleanup incorrectly routed to Velero cleanup; matching Velero Restore was deleted (err=%v)", err)
|
|
}
|
|
}
|
|
|
|
// TestCleanupOnDelete_Velero_DeletesOwnedRestore confirms the dispatcher
|
|
// still routes Velero RestoreJob deletions to the Velero cleanup path.
|
|
func TestCleanupOnDelete_Velero_DeletesOwnedRestore(t *testing.T) {
|
|
apiGroup := strategyv1alpha1.GroupVersion.Group
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
|
}
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: &apiGroup, Kind: strategyv1alpha1.VeleroStrategyKind, Name: "s",
|
|
},
|
|
},
|
|
}
|
|
owned := &velerov1.Restore{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: veleroNamespace,
|
|
Name: "owned",
|
|
Labels: map[string]string{
|
|
backupsv1alpha1.OwningJobNameLabel: "rj",
|
|
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
|
},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj, backup, owned)
|
|
r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
|
|
|
r.cleanupOnDelete(context.Background(), rj)
|
|
|
|
got := &velerov1.Restore{}
|
|
err := c.Get(context.Background(), client.ObjectKeyFromObject(owned), got)
|
|
if err == nil {
|
|
t.Errorf("expected owned Velero Restore to be deleted, but it still exists")
|
|
}
|
|
}
|
|
|
|
func newRestoreJobTestClient(t *testing.T, objs ...client.Object) client.Client {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(s)
|
|
_ = backupsv1alpha1.AddToScheme(s)
|
|
_ = velerov1.AddToScheme(s)
|
|
return clientfake.NewClientBuilder().
|
|
WithScheme(s).
|
|
WithObjects(objs...).
|
|
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.BackupJob{}).
|
|
Build()
|
|
}
|
|
|
|
// TestMarkRestoreJobFailed_DoesNotAppendDuplicateReady locks in the fix:
|
|
// markRestoreJobFailed used to `append` to Status.Conditions, which violates
|
|
// the +listType=map +listMapKey=type contract on the field. Two terminal
|
|
// failures (or a terminal failure on top of a transient Ready=False
|
|
// condition that a previous reconcile already wrote, e.g.
|
|
// WaitingForArchive) used to leave two entries with Type="Ready" - any
|
|
// consumer doing meta.FindStatusCondition reads whichever copy comes first
|
|
// in the slice, not necessarily the latest. SetStatusCondition keeps the
|
|
// list a proper map.
|
|
func TestMarkRestoreJobFailed_DoesNotAppendDuplicateReady(t *testing.T) {
|
|
t.Run("two failures keep exactly one Ready condition", func(t *testing.T) {
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "rj"},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj)
|
|
r := &RestoreJobReconciler{Client: c}
|
|
|
|
if _, err := r.markRestoreJobFailed(context.Background(), rj, "first failure"); err != nil {
|
|
t.Fatalf("first markRestoreJobFailed: %v", err)
|
|
}
|
|
if _, err := r.markRestoreJobFailed(context.Background(), rj, "second failure"); err != nil {
|
|
t.Fatalf("second markRestoreJobFailed: %v", err)
|
|
}
|
|
|
|
readyCount := 0
|
|
for _, cond := range rj.Status.Conditions {
|
|
if cond.Type == "Ready" {
|
|
readyCount++
|
|
}
|
|
}
|
|
if readyCount != 1 {
|
|
t.Fatalf("expected exactly 1 Ready condition, got %d (full slice: %+v)", readyCount, rj.Status.Conditions)
|
|
}
|
|
ready := meta.FindStatusCondition(rj.Status.Conditions, "Ready")
|
|
if ready == nil || ready.Message != "second failure" {
|
|
t.Errorf("expected latest failure message preserved, got %+v", ready)
|
|
}
|
|
})
|
|
|
|
t.Run("transient Ready=False condition is replaced in place", func(t *testing.T) {
|
|
rj := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "rj"},
|
|
Status: backupsv1alpha1.RestoreJobStatus{
|
|
Conditions: []metav1.Condition{
|
|
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "WaitingForArchive", Message: "stale"},
|
|
},
|
|
},
|
|
}
|
|
c := newRestoreJobTestClient(t, rj)
|
|
r := &RestoreJobReconciler{Client: c}
|
|
|
|
if _, err := r.markRestoreJobFailed(context.Background(), rj, "archive deadline exceeded"); err != nil {
|
|
t.Fatalf("markRestoreJobFailed: %v", err)
|
|
}
|
|
if got := len(rj.Status.Conditions); got != 1 {
|
|
t.Fatalf("expected 1 condition, got %d (full slice: %+v)", got, rj.Status.Conditions)
|
|
}
|
|
if rj.Status.Conditions[0].Reason != "RestoreFailed" {
|
|
t.Errorf("expected Reason=RestoreFailed, got %q", rj.Status.Conditions[0].Reason)
|
|
}
|
|
if rj.Status.Conditions[0].Message != "archive deadline exceeded" {
|
|
t.Errorf("expected new message, got %q", rj.Status.Conditions[0].Message)
|
|
}
|
|
})
|
|
}
|