Files
cozystack/internal/backupcontroller/kubernetesstrategy_controller_test.go
T
Andrey Kolkov 0f44cf9e25 feat(backups): add Kubernetes backup strategy for tenant clusters
A tenant Kubernetes application had no backup route in cozy-default, and
none of the existing strategies fits it. Its API state is a key prefix in
the etcd DataStore the whole tenant shares, so the Etcd strategy's snapshot
would roll every tenant cluster back at once; its control-plane CA, service
account key and Talos secrets cannot be regenerated without breaking every
node and credential issued under them; and its PersistentVolumes are
DataVolumes in the tenant namespace that the application's own deletion
removes.

The new Kubernetes strategy backs up the cluster's etcd prefix with a Job
that authenticates with the cluster's own datastore certificate, and the PKI
Secrets and data DataVolumes with Velero. The HelmReleases of the cluster
and its node pools are recorded on the Backup rather than in Velero,
because a label selector for them would also pick up the worker system
disks. A backup warns when a CA would expire before the backup does.

A restore rebuilds a deleted cluster under its old CA, or rolls a live one
back after stopping its control plane and workers, and lets the etcd
prefix and the volumes be restored separately.

Signed-off-by: Andrey Kolkov <androndo@gmail.com>
Assisted-by: LLM
2026-11-10 15:28:34 +07:00

1484 lines
64 KiB
Go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"strconv"
"strings"
"testing"
"time"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
dynamicfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/scheme"
clienttesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)
const (
k8sTestNS = "tenant-a"
k8sTestApp = "c1"
k8sTestRelease = "kubernetes-c1"
)
var kubernetesAppGVR = schema.GroupVersionResource{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Resource: "kubernetes"}
func testCertPEM(t *testing.T, isCA bool, notAfter time.Time) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test"},
NotBefore: notAfter.Add(-time.Hour),
NotAfter: notAfter,
IsCA: isCA,
BasicConstraintsValid: true,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
}
func unstr(gvk schema.GroupVersionKind, ns, name string, fields map[string]any) *unstructured.Unstructured {
u := &unstructured.Unstructured{Object: map[string]any{}}
for k, v := range fields {
u.Object[k] = v
}
u.SetGroupVersionKind(gvk)
u.SetNamespace(ns)
u.SetName(name)
return u
}
func TestKubernetesCertificateWarnings(t *testing.T) {
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
secrets := []corev1.Secret{
{ObjectMeta: metav1.ObjectMeta{Name: "ca"}, Data: map[string][]byte{
"ca.crt": testCertPEM(t, true, now.Add(10*24*time.Hour)),
"ca.key": []byte("not a certificate"),
}},
{ObjectMeta: metav1.ObjectMeta{Name: "leaf-soon"}, Data: map[string][]byte{
"tls.crt": testCertPEM(t, false, now.Add(24*time.Hour)),
}},
{ObjectMeta: metav1.ObjectMeta{Name: "leaf-later"}, Data: map[string][]byte{
"tls.crt": testCertPEM(t, false, now.Add(10*24*time.Hour)),
}},
}
got := kubernetesCertificateWarnings(secrets, now, 30*24*time.Hour, 2*24*time.Hour)
if len(got) != 2 {
t.Fatalf("got %d warnings %v, want 2", len(got), got)
}
if !strings.HasPrefix(got[0], "CA ca/ca.crt") || !strings.HasPrefix(got[1], "certificate leaf-soon/tls.crt") {
t.Fatalf("unexpected warnings %v", got)
}
if got := kubernetesCertificateWarnings(secrets, now, time.Hour, time.Hour); len(got) != 0 {
t.Fatalf("want no warnings inside a short window, got %v", got)
}
}
func TestKubernetesHelmReleaseNames(t *testing.T) {
mdGVK := schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"}
md := func(name, cluster, owner string) unstructured.Unstructured {
u := unstr(mdGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"clusterName": cluster}})
u.SetAnnotations(map[string]string{helmReleaseNameAnnotation: owner})
return *u
}
got := kubernetesHelmReleaseNames([]unstructured.Unstructured{
md("a", k8sTestRelease, "kubernetes-nodes-c1-a"),
md("b", k8sTestRelease, "kubernetes-nodes-c1-b"),
md("legacy", k8sTestRelease, k8sTestRelease),
md("other", "kubernetes-c2", "kubernetes-nodes-c2-a"),
}, k8sTestRelease)
want := []string{k8sTestRelease, "kubernetes-nodes-c1-a", "kubernetes-nodes-c1-b"}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("got %v, want %v", got, want)
}
}
func TestParseKubernetesRestoreOptions(t *testing.T) {
o, err := parseKubernetesRestoreOptions(nil)
if err != nil || !o.restoreEtcd() || !o.restoreVolumes() {
t.Fatalf("defaults: %+v %v", o, err)
}
o, err = parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"volumes":false}`)})
if err != nil || !o.restoreEtcd() || o.restoreVolumes() {
t.Fatalf("etcd only: %+v %v", o, err)
}
if _, err := parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"etcd":false,"volumes":false}`)}); err == nil {
t.Fatal("want an error when both halves are disabled")
}
}
func TestKubernetesVeleroRestoreSpecPhases(t *testing.T) {
strategy := &strategyv1alpha1.Kubernetes{}
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS},
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: k8sTestApp}},
}
secrets, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseSecrets, k8sTestRelease)
if err != nil {
t.Fatal(err)
}
if strings.Join(secrets.IncludedResources, ",") != "secrets" || len(secrets.OrLabelSelectors) != 2 {
t.Fatalf("secrets phase: %+v", secrets)
}
volumes, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseVolumes, k8sTestRelease)
if err != nil {
t.Fatal(err)
}
if volumes.OrLabelSelectors[0].MatchLabels[clusterNameLabel] != k8sTestRelease || volumes.RestorePVs == nil || !*volumes.RestorePVs {
t.Fatalf("volumes phase: %+v", volumes)
}
if _, err := kubernetesVeleroRestoreSpec(strategy, backup, "bogus", k8sTestRelease); err == nil {
t.Fatal("want an error for an unknown phase")
}
}
type kubernetesTestEnv struct {
backup *BackupJobReconciler
restore *RestoreJobReconciler
client client.Client
}
// newKubernetesTestEnv seeds a live tenant cluster: the application, its
// HelmReleases, TenantControlPlane, DataStore, KamajiControlPlane, one worker
// pool and one data DataVolume.
const k8sTestControlPlaneUID = "6b0c4f1e-2d3a-4e5f-8a9b-0c1d2e3f4a5b"
func newKubernetesTestEnv(t *testing.T, objs ...client.Object) *kubernetesTestEnv {
t.Helper()
s := runtime.NewScheme()
_ = scheme.AddToScheme(s)
_ = backupsv1alpha1.AddToScheme(s)
_ = strategyv1alpha1.AddToScheme(s)
_ = velerov1.AddToScheme(s)
listKinds := map[schema.GroupVersionResource]string{
kubernetesAppGVR: "KubernetesList",
helmReleaseGVR: "HelmReleaseList",
tenantControlPlaneGVR: "TenantControlPlaneList",
kamajiDataStoreGVR: "DataStoreList",
kamajiControlPlaneGVR: "KamajiControlPlaneList",
machineDeploymentGVR: "MachineDeploymentList",
machineGVR: "MachineList",
dataVolumeGVR: "DataVolumeList",
deploymentGVR: "DeploymentList",
}
dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(s, listKinds)
seed := func(gvr schema.GroupVersionResource, u *unstructured.Unstructured) {
if err := dyn.Tracker().Create(gvr, u, u.GetNamespace()); err != nil {
t.Fatalf("seed %s: %v", u.GetName(), err)
}
}
appGVK := schema.GroupVersionKind{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Kind: kubernetesAppKind}
seed(kubernetesAppGVR, unstr(appGVK, k8sTestNS, k8sTestApp, map[string]any{"spec": map[string]any{}}))
hrGVK := schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}
for _, name := range []string{k8sTestRelease, "kubernetes-nodes-c1-md0"} {
hr := unstr(hrGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"interval": "5m"}})
hr.SetLabels(map[string]string{"apps.cozystack.io/application.name": name, "velero.io/backup-name": "old"})
seed(helmReleaseGVR, hr)
}
tcp := unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{
"status": map[string]any{"storage": map[string]any{
"driver": "etcd",
"dataStoreName": "tenant-a",
"setup": map[string]any{"schema": "tenant_a_kubernetes_c1"},
"certificate": map[string]any{"secretName": k8sTestRelease + "-datastore-certificate"},
}},
})
tcp.SetUID(k8sTestControlPlaneUID)
seed(tenantControlPlaneGVR, tcp)
seed(kamajiDataStoreGVR, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "DataStore"}, "", "tenant-a", map[string]any{
"spec": map[string]any{"endpoints": []any{"etcd.tenant-a.svc:2379"}},
}))
seed(kamajiControlPlaneGVR, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{
"spec": map[string]any{"replicas": int64(2)},
}))
md := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"}, k8sTestNS, k8sTestRelease+"-md0", map[string]any{
"spec": map[string]any{"clusterName": k8sTestRelease, "replicas": int64(3)},
})
md.SetAnnotations(map[string]string{helmReleaseNameAnnotation: "kubernetes-nodes-c1-md0"})
seed(machineDeploymentGVR, md)
dv := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-1234", nil)
dv.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease})
seed(dataVolumeGVR, dv)
seed(deploymentGVR, unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease+"-cluster-autoscaler", map[string]any{
"spec": map[string]any{"replicas": int64(1)},
}))
base := []client.Object{
&velerov1.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "tenant-a.bj1-xyz"},
Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseCompleted},
},
&strategyv1alpha1.Kubernetes{
ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-kubernetes"},
Spec: strategyv1alpha1.KubernetesSpec{
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
Name: "etcd",
Image: "img",
Args: []string{"{{ .Mode }} {{ .ObjectKey }} {{ .Datastore.Prefix }} {{ .Datastore.Endpoints }} {{ .Datastore.CertificateSecret }} {{ .Datastore.SourcePrefix }}"},
}}}},
Velero: strategyv1alpha1.VeleroTemplate{Spec: velerov1.BackupSpec{
IncludedNamespaces: []string{"{{ .Application.metadata.namespace }}"},
TTL: metav1.Duration{Duration: 720 * time.Hour},
}},
},
},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-ca"}, Data: map[string][]byte{
"ca.crt": testCertPEM(t, true, time.Now().Add(24*time.Hour)),
}},
}
c := clientfake.NewClientBuilder().WithScheme(s).
WithObjects(append(base, objs...)...).
WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}).
Build()
mapper := &mockRESTMapper{mapping: &meta.RESTMapping{Resource: kubernetesAppGVR, GroupVersionKind: appGVK, Scope: meta.RESTScopeNamespace}}
return &kubernetesTestEnv{
backup: &BackupJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)},
restore: &RestoreJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)},
client: c,
}
}
func k8sResolved() *ResolvedBackupConfig {
return &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new("strategy.backups.cozystack.io"), Kind: strategyv1alpha1.KubernetesStrategyKind, Name: "cozy-default-kubernetes"},
Parameters: map[string]string{"p": "v"},
}
}
func TestReconcileKubernetesBackup(t *testing.T) {
ctx := context.Background()
bj := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"},
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp,
}},
}
env := newKubernetesTestEnv(t, bj)
get := func() *backupsv1alpha1.BackupJob {
out := &backupsv1alpha1.BackupJob{}
if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil {
t.Fatal(err)
}
return out
}
step := func() {
t.Helper()
if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil {
t.Fatal(err)
}
}
step() // StartedAt
step() // certificates
cond := meta.FindStatusCondition(get().Status.Conditions, kubernetesCondCertificates)
if cond == nil || cond.Status != metav1.ConditionTrue || !strings.Contains(cond.Message, k8sTestRelease+"-ca/ca.crt") {
t.Fatalf("certificate condition: %+v", cond)
}
step() // dump Job
job := &batchv1.Job{}
if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1-backup"}, job); err != nil {
t.Fatalf("dump Job: %v", err)
}
args := job.Spec.Template.Spec.Containers[0].Args[0]
want := "backup tenant-a/c1/bj1.etcd.json tenant_a_kubernetes_c1 https://etcd.tenant-a.svc:2379 kubernetes-c1-datastore-certificate tenant_a_kubernetes_c1"
if args != want {
t.Fatalf("rendered args %q, want %q", args, want)
}
step()
vbs := &velerov1.BackupList{}
_ = env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"})
if len(vbs.Items) != 0 {
t.Fatal("Velero Backup created before the dump Job completed")
}
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
if err := env.client.Status().Update(ctx, job); err != nil {
t.Fatal(err)
}
// A claim the CSI driver created without a DataVolume is still one an
// in-place restore has to clear before Velero writes it back.
if err := env.client.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
Namespace: k8sTestNS, Name: "pvc-plain", Labels: map[string]string{clusterNameLabel: k8sTestRelease},
}}); err != nil {
t.Fatal(err)
}
step()
if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}); err != nil || len(vbs.Items) != 1 {
t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err)
}
// A volume created while Velero runs may be missing from its backup, so a
// restore of this backup must never delete it.
late := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-late", nil)
late.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease})
if _, err := env.backup.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Create(ctx, late, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if ns := vbs.Items[0].Spec.IncludedNamespaces; len(ns) != 1 || ns[0] != k8sTestNS {
t.Fatalf("Velero spec not rendered: %v", ns)
}
talosCA := &corev1.Secret{}
_ = env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}, talosCA)
if talosCA.Labels[helmInstanceLabel] != k8sTestRelease {
t.Fatalf("talos CA not labelled for the Velero selector: %v", talosCA.Labels)
}
vb := vbs.Items[0]
vb.Status.Phase = velerov1.BackupPhaseCompleted
if err := env.client.Update(ctx, &vb); err != nil {
t.Fatal(err)
}
// A pool that is suspended while the backup runs must not come back
// suspended when the cluster is rebuilt from it.
if _, err := env.backup.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(ctx, "kubernetes-nodes-c1-md0", types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil {
t.Fatal(err)
}
step()
if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded || got.Status.BackupRef == nil {
t.Fatalf("BackupJob not succeeded: %+v", got.Status)
}
backup := &backupsv1alpha1.Backup{}
if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1"}, backup); err != nil {
t.Fatal(err)
}
md := backup.Spec.DriverMetadata
if md[veleroBackupNameMetadataKey] != vb.Name || md[kubernetesDataVolumesMetaKey] != "pvc-1234,pvc-plain" ||
md[kubernetesDatastorePrefixMetaKey] != "tenant_a_kubernetes_c1" || md[kubernetesParamPrefix+"p"] != "v" ||
md[kubernetesCertWarningsMetaKey] == "" || md[kubernetesControlPlaneUIDMetaKey] != k8sTestControlPlaneUID {
t.Fatalf("driver metadata: %v", md)
}
releases, err := kubernetesBackupHelmReleases(backup)
if err != nil || len(releases) != 2 {
t.Fatalf("recorded HelmReleases: %v", releases)
}
raw, _ := json.Marshal(releases)
if strings.Contains(string(raw), "velero.io/") || strings.Contains(string(raw), "suspend") {
t.Fatalf("Velero labels or the suspend flag leaked into the recorded HelmReleases: %s", raw)
}
}
func k8sBackupForRestore() *backupsv1alpha1.Backup {
return &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1"},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp},
StrategyRef: k8sResolved().StrategyRef,
DriverMetadata: map[string]string{
veleroBackupNameMetadataKey: "tenant-a.bj1-xyz",
kubernetesObjectMetaKey: "tenant-a/c1/bj1.etcd.json",
kubernetesDatastorePrefixMetaKey: "tenant_a_kubernetes_c1",
kubernetesControlPlaneUIDMetaKey: k8sTestControlPlaneUID,
kubernetesDatastoreEndpointsMetaKey: "https://etcd.tenant-a.svc:2379",
kubernetesDataVolumesMetaKey: "pvc-1234",
kubernetesHelmReleasesMetaKey: `[{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-c1"},"spec":{"interval":"5m"}}]`,
},
},
}
}
// k8sRestoreRun drives one RestoreJob through the test environment.
type k8sRestoreRun struct {
t *testing.T
env *kubernetesTestEnv
rj *backupsv1alpha1.RestoreJob
backup *backupsv1alpha1.Backup
}
func newK8sRestoreRun(t *testing.T, options string, backup *backupsv1alpha1.Backup) *k8sRestoreRun {
t.Helper()
rj := &backupsv1alpha1.RestoreJob{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj"},
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
}
if options != "" {
rj.Spec.Options = &runtime.RawExtension{Raw: []byte(options)}
}
return &k8sRestoreRun{t: t, env: newKubernetesTestEnv(t, backup, rj), rj: rj, backup: backup}
}
func (r *k8sRestoreRun) get() *backupsv1alpha1.RestoreJob {
r.t.Helper()
out := &backupsv1alpha1.RestoreJob{}
if err := r.env.client.Get(context.Background(), client.ObjectKeyFromObject(r.rj), out); err != nil {
r.t.Fatal(err)
}
return out
}
func (r *k8sRestoreRun) step() {
r.t.Helper()
if _, err := r.env.restore.reconcileKubernetesRestore(context.Background(), r.get(), r.backup); err != nil {
r.t.Fatal(err)
}
}
func (r *k8sRestoreRun) finishJob(name string, condType batchv1.JobConditionType) {
r.t.Helper()
job := &batchv1.Job{}
if err := r.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: name}, job); err != nil {
r.t.Fatalf("Job %s: %v", name, err)
}
job.Status.Conditions = []batchv1.JobCondition{{Type: condType, Status: corev1.ConditionTrue, Message: "boom"}}
if err := r.env.client.Status().Update(context.Background(), job); err != nil {
r.t.Fatal(err)
}
}
func (r *k8sRestoreRun) finishVeleroRestore(phase velerov1.RestorePhase) {
r.t.Helper()
list := &velerov1.RestoreList{}
if err := r.env.client.List(context.Background(), list); err != nil || len(list.Items) == 0 {
r.t.Fatalf("Velero Restores: %v %v", len(list.Items), err)
}
vr := list.Items[len(list.Items)-1]
vr.Status.Phase = phase
if err := r.env.client.Update(context.Background(), &vr); err != nil {
r.t.Fatal(err)
}
}
func (r *k8sRestoreRun) obj(gvr schema.GroupVersionResource, name string) *unstructured.Unstructured {
r.t.Helper()
u, err := r.env.restore.Interface.Resource(gvr).Namespace(k8sTestNS).Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
r.t.Fatalf("get %s %s: %v", gvr.Resource, name, err)
}
return u
}
func (r *k8sRestoreRun) replicas(gvr schema.GroupVersionResource, name string) (int64, bool) {
n, found, _ := unstructured.NestedInt64(r.obj(gvr, name).Object, "spec", "replicas")
return n, found
}
func (r *k8sRestoreRun) suspended(name string) bool {
s, _, _ := unstructured.NestedBool(r.obj(helmReleaseGVR, name).Object, "spec", "suspend")
return s
}
func (r *k8sRestoreRun) addMachine(name string) {
r.t.Helper()
m := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "Machine"}, k8sTestNS, name, nil)
m.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease})
if _, err := r.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Create(context.Background(), m, metav1.CreateOptions{}); err != nil {
r.t.Fatal(err)
}
}
// verifyAndCapture runs the preflight and the mode decision.
func (r *k8sRestoreRun) verifyAndCapture(etcd bool) {
r.t.Helper()
r.step() // StartedAt
r.step() // preflight
if etcd {
r.finishJob("rj-verify", batchv1.JobComplete)
r.step()
}
if !meta.IsStatusConditionTrue(r.get().Status.Conditions, kubernetesCondVerified) {
r.t.Fatalf("backup not verified: %+v", r.get().Status)
}
r.step() // capture
}
func TestReconcileKubernetesRestoreInPlaceDefault(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
run.addMachine("w-1")
run.verifyAndCapture(true)
run.step() // stop: a worker is still there
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 {
t.Fatal("control plane scaled down while a worker Machine still needs it to delete its Node")
}
if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 0 {
t.Fatalf("worker pool not scaled down: %d", n)
}
if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 {
t.Fatalf("autoscaler not scaled down: %d", n)
}
if _, ok := run.obj(machineGVR, "w-1").GetAnnotations()[machineExcludeDrainAnnotation]; !ok {
t.Fatal("Machine not released from draining")
}
if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondStopped); c == nil || c.Status != metav1.ConditionFalse || !strings.Contains(c.Message, "Machine") {
t.Fatalf("stop progress not reported: %+v", c)
}
if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil {
t.Fatal("data DataVolume deleted while a worker may still mount it")
}
if err := run.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Delete(context.Background(), "w-1", metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
run.step() // stop settles
if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err == nil {
t.Fatal("backed-up data DataVolume not deleted once the workers were gone")
}
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatal("control plane not scaled down after the workers were gone")
}
if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) {
t.Fatal("cluster not reported stopped")
}
run.step() // etcd Job
run.finishJob("rj-restore", batchv1.JobComplete)
run.step() // etcd done
run.step() // volumes Velero Restore
run.finishVeleroRestore(velerov1.RestorePhaseCompleted)
run.step() // volumes done
run.step() // resume
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded {
t.Fatalf("RestoreJob not succeeded: %+v", got.Status)
}
for _, c := range []struct {
gvr schema.GroupVersionResource
name string
want int64
}{
{kamajiControlPlaneGVR, k8sTestRelease, 2},
{machineDeploymentGVR, k8sTestRelease + "-md0", 3},
{deploymentGVR, k8sTestRelease + "-cluster-autoscaler", 1},
} {
if n, _ := run.replicas(c.gvr, c.name); n != c.want {
t.Errorf("%s %s replicas = %d, want %d", c.gvr.Resource, c.name, n, c.want)
}
if _, left := run.obj(c.gvr, c.name).GetAnnotations()[kubernetesReplicasBeforeAnnotation]; left {
t.Errorf("%s %s keeps the restore's replica record", c.gvr.Resource, c.name)
}
}
if run.suspended(k8sTestRelease) || run.suspended("kubernetes-nodes-c1-md0") {
t.Fatal("HelmRelease left suspended")
}
}
func TestReconcileKubernetesRestoreInPlaceEtcdOnly(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
run.step() // stop
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatalf("control plane not scaled down: %d", n)
}
if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 {
t.Fatalf("workers touched by an etcd-only restore: %d", n)
}
run.step()
run.finishJob("rj-restore", batchv1.JobComplete)
run.step()
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded {
t.Fatalf("RestoreJob not succeeded: %+v", got.Status)
}
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 {
t.Fatalf("control plane replicas not restored: %d", n)
}
list := &velerov1.RestoreList{}
_ = run.env.client.List(context.Background(), list)
if len(list.Items) != 0 {
t.Fatal("an etcd-only restore created a Velero Restore")
}
}
func TestReconcileKubernetesRestoreKeepsPreSuspendedRelease(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
patch := []byte(`{"spec":{"suspend":true}}`)
if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(context.Background(), "kubernetes-nodes-c1-md0", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
t.Fatal(err)
}
run.verifyAndCapture(true)
run.step()
run.step()
run.finishJob("rj-restore", batchv1.JobComplete)
run.step()
run.step()
if run.get().Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded {
t.Fatalf("RestoreJob not succeeded: %+v", run.get().Status)
}
if !run.suspended("kubernetes-nodes-c1-md0") {
t.Fatal("a HelmRelease suspended before the restore was resumed by it")
}
if run.suspended(k8sTestRelease) {
t.Fatal("the cluster HelmRelease the restore suspended was left suspended")
}
}
func TestReconcileKubernetesRestoreEtcdFailureLeavesControlPlaneStopped(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
run.step()
run.step()
run.addJobPod("rj-restore", kubernetesJobWriting)
run.finishJob("rj-restore", batchv1.JobFailed)
run.step()
got := run.get()
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) {
t.Fatalf("want a failed restore with the etcd write marked failed: %+v", got.Status)
}
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatal("control plane brought back up on a possibly partial etcd prefix")
}
if !strings.Contains(got.Status.Message+fmtConditions(got), ".pre-restore") {
t.Fatalf("failure does not point at the pre-restore copy: %+v", got.Status)
}
run.env.restore.cleanupKubernetesRestore(context.Background(), got)
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatal("deleting the RestoreJob brought the control plane up on a possibly partial prefix")
}
}
func fmtConditions(rj *backupsv1alpha1.RestoreJob) string {
var b strings.Builder
for _, c := range rj.Status.Conditions {
b.WriteString(c.Message)
}
return b.String()
}
func TestCleanupKubernetesRestoreResumesStoppedCluster(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
run.verifyAndCapture(true)
run.step() // stop: workers gone, volumes deleted, control plane stopped
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatalf("control plane not stopped: %d", n)
}
run.env.restore.cleanupKubernetesRestore(context.Background(), run.get())
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 {
t.Fatalf("deleting the RestoreJob left the control plane at %d replicas", n)
}
if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 {
t.Fatalf("deleting the RestoreJob left the worker pool at %d replicas", n)
}
if run.suspended(k8sTestRelease) {
t.Fatal("deleting the RestoreJob left the HelmRelease suspended")
}
}
func TestReconcileKubernetesRestoreStopDeadline(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
run.addMachine("w-1")
run.verifyAndCapture(true)
run.step() // stop starts, a worker remains
rj := run.get()
cond := meta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped)
cond.LastTransitionTime = metav1.NewTime(time.Now().Add(-restorePreparationDeadline - time.Minute))
if err := run.env.client.Status().Update(context.Background(), rj); err != nil {
t.Fatal(err)
}
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("a stop that never settles must fail the restore: %+v", got.Status)
}
if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 {
t.Fatalf("worker pool not resumed after the stop timed out: %d", n)
}
if run.suspended(k8sTestRelease) {
t.Fatal("HelmRelease left suspended after the stop timed out")
}
}
func TestReconcileKubernetesRestoreVerifiesBeforeChangingAnything(t *testing.T) {
backup := k8sBackupForRestore()
backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "expired"
run := newK8sRestoreRun(t, "", backup)
run.step()
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("a missing Velero backup must fail the restore: %+v", got.Status)
}
if run.suspended(k8sTestRelease) {
t.Fatal("cluster touched before the backup was verified")
}
if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil {
t.Fatal("data DataVolume deleted before the backup was verified")
}
backup = k8sBackupForRestore()
backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "partial"
run = newK8sRestoreRun(t, "", backup)
if err := run.env.client.Create(context.Background(), &velerov1.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "partial"},
Status: velerov1.BackupStatus{Phase: velerov1.BackupPhasePartiallyFailed},
}); err != nil {
t.Fatal(err)
}
run.step()
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("a Velero backup that did not complete must fail the restore: %+v", got.Status)
}
run = newK8sRestoreRun(t, "", k8sBackupForRestore())
run.step()
run.step()
run.finishJob("rj-verify", batchv1.JobFailed)
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("a missing etcd dump must fail the restore: %+v", got.Status)
}
if run.suspended(k8sTestRelease) {
t.Fatal("cluster touched before the dump was verified")
}
}
func k8sBackupOfDeletedCluster() *backupsv1alpha1.Backup {
backup := k8sBackupForRestore()
backup.Spec.ApplicationRef.Name = "gone"
backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = `[` +
`{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-gone"},"spec":{"interval":"5m"}},` +
`{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-nodes-gone-md0"},"spec":{"interval":"5m"}}]`
return backup
}
// TestReconcileKubernetesRestoreRecreate rebuilds a deleted cluster whose new
// TenantControlPlane gets a fresh, UID-keyed schema: the dump must be written
// under that schema, and the worker pools come back only after it is.
func TestReconcileKubernetesRestoreRecreate(t *testing.T) {
ctx := context.Background()
run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster())
run.verifyAndCapture(true)
if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeRecreate || state.Release != "kubernetes-gone" {
t.Fatalf("want recreate mode, got %+v", state)
}
run.step() // secrets
run.finishVeleroRestore(velerov1.RestorePhaseCompleted)
run.step()
run.step() // volumes
run.finishVeleroRestore(velerov1.RestorePhaseCompleted)
run.step()
run.step() // cluster HelmRelease, waiting for the TenantControlPlane
dyn := run.env.restore.Interface
if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-gone", metav1.GetOptions{}); err != nil {
t.Fatalf("cluster HelmRelease not recreated: %v", err)
}
if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err == nil {
t.Fatal("worker pool recreated before the etcd state was written back")
}
if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondRecreated); c == nil || c.Status != metav1.ConditionFalse {
t.Fatalf("wait for the new control plane not reported: %+v", c)
}
run.step() // a later poll, the control plane still coming up
if ann := run.obj(helmReleaseGVR, "kubernetes-gone").GetAnnotations(); ann["reconcile.fluxcd.io/forceAt"] != "" {
t.Fatalf("the cluster HelmRelease the rebuild created is forced through a Helm upgrade on a poll: %v", ann)
}
const newSchema = "0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10"
if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{
"status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": newSchema}}},
}), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{
"spec": map[string]any{"replicas": int64(2)},
}), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
run.step() // control plane is up
run.step() // stop it for the etcd write
if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 0 {
t.Fatalf("new control plane not stopped before the etcd write: %d", n)
}
run.step() // etcd Job
job := &batchv1.Job{}
if err := run.env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err != nil {
t.Fatal(err)
}
args := job.Spec.Template.Spec.Containers[0].Args[0]
if !strings.Contains(args, " "+newSchema+" ") || !strings.HasSuffix(args, " tenant_a_kubernetes_c1") || !strings.Contains(args, "kubernetes-gone-datastore-certificate") {
t.Fatalf("restore Job does not write the dump under the new cluster's schema: %q", args)
}
run.finishJob("rj-restore", batchv1.JobComplete)
run.step()
run.step() // resume, worker pools
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded {
t.Fatalf("rebuild not succeeded: %+v", got.Status)
}
if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 {
t.Fatalf("new control plane not started again: %d", n)
}
if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err != nil {
t.Fatalf("worker pool not recreated: %v", err)
}
}
func TestReconcileKubernetesRestoreRecreateSecretsFailure(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster())
run.verifyAndCapture(true)
run.step() // secrets restore
list := &velerov1.RestoreList{}
if err := run.env.client.List(context.Background(), list); err != nil || len(list.Items) != 1 {
t.Fatalf("Velero Restores: %d %v", len(list.Items), err)
}
vr := list.Items[0]
if vr.Labels[kubernetesRestorePhaseLabel] != kubernetesPhaseSecrets || vr.Spec.BackupName != "tenant-a.bj1-xyz" || vr.Spec.ResourceModifier == nil {
t.Fatalf("secrets Restore: %+v", vr)
}
cm := &corev1.ConfigMap{}
if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: veleroNamespace, Name: vr.Spec.ResourceModifier.Name}, cm); err != nil {
t.Fatal(err)
}
if !strings.Contains(cm.Data["resource-modifier-rules.yaml"], "ownerReferences: null") {
t.Fatalf("modifiers do not strip ownerReferences: %s", cm.Data["resource-modifier-rules.yaml"])
}
run.finishVeleroRestore(velerov1.RestorePhaseFailed)
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("a failed Velero Restore must fail the RestoreJob: %+v", got.Status)
}
}
func TestReconcileKubernetesRestoreRecreateRejectsBadReleaseRecord(t *testing.T) {
backup := k8sBackupForRestore()
backup.Spec.ApplicationRef.Name = "gone"
backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = "not json"
run := newK8sRestoreRun(t, "", backup)
run.verifyAndCapture(true)
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("an unreadable HelmRelease record must fail the rebuild: %+v", got.Status)
}
}
func TestReconcileKubernetesRestoreRejectsRename(t *testing.T) {
backup := k8sBackupForRestore()
run := newK8sRestoreRun(t, "", backup)
rj := run.get()
rj.Spec.TargetApplicationRef = &corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: "other"}
if _, err := run.env.restore.reconcileKubernetesRestore(context.Background(), rj, backup); err != nil {
t.Fatal(err)
}
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("restore into another application must fail: %+v", got.Status)
}
}
func TestResolveKubernetesDatastoreRejectsNonEtcd(t *testing.T) {
env := newKubernetesTestEnv(t)
dyn := env.backup.Interface
tcp, _ := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(context.Background(), k8sTestRelease, metav1.GetOptions{})
_ = unstructured.SetNestedField(tcp.Object, "PostgreSQL", "status", "storage", "driver")
if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(context.Background(), tcp, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
if _, err := resolveKubernetesDatastore(context.Background(), dyn, k8sTestNS, k8sTestApp); err == nil {
t.Fatal("want an error for a non-etcd datastore")
}
}
func TestCleanupKubernetesBackupGivesUpAfterRepeatedFailures(t *testing.T) {
ctx := context.Background()
backup := k8sBackupForRestore()
backup.Annotations = map[string]string{kubernetesCleanupAttemptsAnnotation: strconv.Itoa(kubernetesCleanupMaxAttempts - 1)}
failed := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1-cleanup"},
Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}},
}
env := newKubernetesTestEnv(t, backup, failed)
br := &BackupReconciler{Client: env.client, Scheme: env.backup.Scheme, Recorder: record.NewFakeRecorder(10)}
res, err := br.cleanupKubernetesBackup(ctx, backup)
if err != nil {
t.Fatal(err)
}
if res.RequeueAfter != 0 {
t.Fatalf("cleanup keeps retrying after %d failures: %+v", kubernetesCleanupMaxAttempts, res)
}
}
func (r *k8sRestoreRun) addJobPod(job, message string) {
r.t.Helper()
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: job + "-pod", Labels: map[string]string{"job-name": job}},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{
Name: "etcd",
State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Message: message + "\n"}},
}}},
}
if err := r.env.client.Create(context.Background(), pod); err != nil {
r.t.Fatal(err)
}
}
func TestReconcileKubernetesRestoreFailureBeforeWriteResumes(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
run.step() // stop
run.step() // etcd Job
run.addJobPod("rj-restore", "")
run.finishJob("rj-restore", batchv1.JobFailed)
run.step()
got := run.get()
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) {
t.Fatalf("a Job that never touched the prefix must fail without marking it partial: %+v", got.Status)
}
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 {
t.Fatalf("control plane left stopped although the prefix was never changed: %d", n)
}
}
func TestCleanupKubernetesRestoreKeepsClusterStoppedWhileRestoreJobRuns(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
run.step() // stop
run.step() // etcd Job, still running
run.env.restore.cleanupKubernetesRestore(context.Background(), run.get())
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatal("deleting the RestoreJob brought the control plane up while the restore Job may be rewriting the prefix")
}
job := &batchv1.Job{}
if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err == nil {
t.Fatal("the running restore Job was left to keep writing")
}
}
func TestReconcileKubernetesRestoreIgnoresModeAnnotation(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
rj := run.get()
rj.Annotations = map[string]string{"kubernetes.strategy.backups.cozystack.io/restore-state": `{"mode":"recreate","release":"kubernetes-c1"}`}
if err := run.env.client.Update(context.Background(), rj); err != nil {
t.Fatal(err)
}
run.verifyAndCapture(true)
if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeInPlace {
t.Fatalf("a live cluster must be restored in place whatever the RestoreJob claims: %+v", state)
}
}
func TestReconcileKubernetesBackupDiscardsDumpWhenVeleroFails(t *testing.T) {
ctx := context.Background()
bj := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj2"},
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp,
}},
}
env := newKubernetesTestEnv(t, bj)
step := func() {
t.Helper()
got := &backupsv1alpha1.BackupJob{}
if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), got); err != nil {
t.Fatal(err)
}
if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil {
t.Fatal(err)
}
}
step()
step()
step()
dump := &batchv1.Job{}
if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-backup"}, dump); err != nil {
t.Fatal(err)
}
dump.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
if err := env.client.Status().Update(ctx, dump); err != nil {
t.Fatal(err)
}
step()
vbs := &velerov1.BackupList{}
if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj2"}); err != nil || len(vbs.Items) != 1 {
t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err)
}
vb := vbs.Items[0]
vb.Status.Phase = velerov1.BackupPhaseFailed
if err := env.client.Update(ctx, &vb); err != nil {
t.Fatal(err)
}
step()
got := &backupsv1alpha1.BackupJob{}
_ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got)
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("BackupJob not failed: %+v", got.Status)
}
discard := &batchv1.Job{}
if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-discard"}, discard); err != nil {
t.Fatalf("the orphaned etcd dump is not scheduled for deletion: %v", err)
}
if args := discard.Spec.Template.Spec.Containers[0].Args[0]; !strings.HasPrefix(args, "cleanup tenant-a/c1/bj2.etcd.json") {
t.Fatalf("discard Job does not delete this run's dump: %q", args)
}
}
func TestReconcileKubernetesRestoreRecreateFailureBeforeWriteStartsControlPlane(t *testing.T) {
ctx := context.Background()
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupOfDeletedCluster())
run.verifyAndCapture(true)
run.step() // secrets
run.finishVeleroRestore(velerov1.RestorePhaseCompleted)
run.step()
run.step() // cluster HelmRelease
dyn := run.env.restore.Interface
if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{
"status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": "uid-1"}}},
}), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{
"spec": map[string]any{"replicas": int64(2)},
}), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
run.step() // control plane is up
run.step() // stopped
run.step() // etcd Job
run.addJobPod("rj-restore", "")
run.finishJob("rj-restore", batchv1.JobFailed)
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("rebuild not failed: %+v", got.Status)
}
if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 {
t.Fatalf("the rebuilt control plane is left stopped although its prefix was never changed: %d", n)
}
}
func TestReconcileKubernetesRestoreJobThatNeverRanResumes(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
run.step() // stop
run.step() // etcd Job
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{
Name: "etcd",
State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}},
}}},
}
if err := run.env.client.Create(context.Background(), pod); err != nil {
t.Fatal(err)
}
run.step() // still waiting on the Job
if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) {
t.Fatal("a container that never ran was taken for a started Job")
}
run.finishJob("rj-restore", batchv1.JobFailed)
run.step()
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 {
t.Fatalf("control plane left stopped after a restore Job that never ran: %d", n)
}
}
func TestReconcileKubernetesRestoreJobPodsGoneAfterStart(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
run.step() // stop
run.step() // etcd Job
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{
Name: "etcd",
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}},
}}},
}
if err := run.env.client.Create(context.Background(), pod); err != nil {
t.Fatal(err)
}
run.step()
if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) {
t.Fatal("a running restore Job was not recorded as started")
}
// The Job deadline deletes its Pods and fails it.
if err := run.env.client.Delete(context.Background(), pod); err != nil {
t.Fatal(err)
}
run.finishJob("rj-restore", batchv1.JobFailed)
run.step()
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 {
t.Fatal("control plane brought up after a restore Job that ran and left no trace of how far it got")
}
}
func TestReconcileKubernetesRestoreRetriesStopAfterAPIError(t *testing.T) {
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
run.verifyAndCapture(true)
failed := false
run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("patch", "kamajicontrolplanes", func(clienttesting.Action) (bool, runtime.Object, error) {
if failed {
return false, nil, nil
}
failed = true
return true, nil, fmt.Errorf("etcdserver: request timed out")
})
run.step()
got := run.get()
if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("one API error aborted the restore: %+v", got.Status)
}
if c := meta.FindStatusCondition(got.Status.Conditions, kubernetesCondStopped); c == nil || !strings.Contains(c.Message, "API error") {
t.Fatalf("the retried API error is not reported: %+v", c)
}
run.step()
if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) {
t.Fatal("the stop did not go through once the API answered again")
}
}
// driveRecreateToEtcdJob runs a rebuild up to the etcd restore Job, the new
// TenantControlPlane reporting schema.
func driveRecreateToEtcdJob(t *testing.T, run *k8sRestoreRun, schemaName string) {
t.Helper()
ctx := context.Background()
run.verifyAndCapture(true)
run.step() // secrets
run.finishVeleroRestore(velerov1.RestorePhaseCompleted)
run.step()
run.step() // volumes
run.finishVeleroRestore(velerov1.RestorePhaseCompleted)
run.step()
run.step() // cluster HelmRelease
dyn := run.env.restore.Interface
if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{
"status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": schemaName}}},
}), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{
"spec": map[string]any{"replicas": int64(2)},
}), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
run.step() // control plane is up
run.step() // stopped
run.step() // etcd Job
}
func TestReconcileKubernetesRestoreRecreateForcesSurvivingPool(t *testing.T) {
ctx := context.Background()
run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster())
// Deleting only the Kubernetes application leaves its KubernetesNodes
// pool's HelmRelease behind, while the pool's MachineDeployment went
// with the CAPI Cluster.
pool := unstr(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, k8sTestNS, "kubernetes-nodes-gone-md0", map[string]any{"spec": map[string]any{"interval": "5m"}})
if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Create(ctx, pool, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
driveRecreateToEtcdJob(t, run, "uid-1")
run.finishJob("rj-restore", batchv1.JobComplete)
run.step()
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded {
t.Fatalf("rebuild not succeeded: %+v", got.Status)
}
ann := run.obj(helmReleaseGVR, "kubernetes-nodes-gone-md0").GetAnnotations()
if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/forceAt"] != ann["reconcile.fluxcd.io/requestedAt"] {
t.Fatalf("the surviving pool is not forced to recreate its MachineDeployment: %v", ann)
}
}
func TestReconcileKubernetesBackupRefusesClusterStoppedByRestore(t *testing.T) {
ctx := context.Background()
bj := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj3"},
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp,
}},
}
env := newKubernetesTestEnv(t, bj)
patch := []byte(`{"metadata":{"annotations":{"` + kubernetesReplicasBeforeAnnotation + `":"2"}},"spec":{"replicas":0}}`)
if _, err := env.backup.Interface.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Patch(ctx, k8sTestRelease, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
t.Fatal(err)
}
for range 2 {
got := &backupsv1alpha1.BackupJob{}
_ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got)
if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil {
t.Fatal(err)
}
}
got := &backupsv1alpha1.BackupJob{}
_ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got)
if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Fatalf("a cluster stopped by an unfinished restore was backed up: %+v", got.Status)
}
if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj3-backup"}, &batchv1.Job{}); err == nil {
t.Fatal("the etcd prefix of a stopped cluster was dumped")
}
}
func TestStopKubernetesClusterWaitsForTerminatingControlPlane(t *testing.T) {
ctx := context.Background()
run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore())
cp := unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease, map[string]any{
"spec": map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"kamaji.clastix.io/name": k8sTestRelease}}},
"status": map[string]any{"replicas": int64(0)},
})
if _, err := run.env.restore.Interface.Resource(deploymentGVR).Namespace(k8sTestNS).Create(ctx, cp, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "apiserver-0", Labels: map[string]string{"kamaji.clastix.io/name": k8sTestRelease}}}
if err := run.env.client.Create(ctx, pod); err != nil {
t.Fatal(err)
}
run.verifyAndCapture(true)
run.step()
if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) {
t.Fatal("cluster reported stopped while a control-plane Pod is still terminating")
}
if err := run.env.client.Delete(ctx, pod); err != nil {
t.Fatal(err)
}
run.step()
if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) {
t.Fatal("cluster not reported stopped once its control plane was gone")
}
}
func TestKubernetesJobReachedWriteConservativeBranches(t *testing.T) {
for _, tc := range []struct {
name string
status corev1.ContainerStatus
want bool
}{
{"running", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, true},
{"restarted after writing", corev1.ContainerStatus{Name: "etcd",
State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}},
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Message: kubernetesJobWriting}}}, true},
{"never ran", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, false},
{"failed before writing", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1}}}, false},
} {
t.Run(tc.name, func(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "p", Labels: map[string]string{"job-name": "rj-restore"}},
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{tc.status}},
}
if err := run.env.client.Create(context.Background(), pod); err != nil {
t.Fatal(err)
}
got, err := run.env.restore.kubernetesJobReachedWrite(context.Background(), run.get(), "rj-restore")
if err != nil || got != tc.want {
t.Fatalf("reached write = %v (%v), want %v", got, err, tc.want)
}
})
}
}
// TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed covers a Cluster
// API that no longer serves the version the driver lists. Reading that as a
// cluster without workers would let the volumes path delete DataVolumes that
// running workers still mount.
func TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed(t *testing.T) {
for _, tc := range []struct{ resource, kind string }{
{"machinedeployments", "MachineDeployment"},
{"machines", "Machine"},
} {
t.Run(tc.resource, func(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
run.verifyAndCapture(true)
run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", tc.resource, func(clienttesting.Action) (bool, runtime.Object, error) {
return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: tc.kind}, SearchedVersions: []string{"v1beta1"}}
})
run.step()
got := run.get()
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "Cluster API is not served") {
t.Fatalf("restore not failed on an unserved Cluster API: %+v", got.Status)
}
if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil {
t.Fatalf("backed-up DataVolume deleted: %v", err)
}
if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 {
t.Fatalf("control plane left at %d replicas", n)
}
if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 {
t.Fatalf("worker pool left at %d replicas", n)
}
if run.suspended(k8sTestRelease) {
t.Fatal("cluster HelmRelease left suspended")
}
})
}
}
// TestReconcileKubernetesRestoreRefusesRecreatedCluster covers an application
// deleted after the backup and created again under the same name. Its new CA
// and service-account key come back only through a rebuild, so an in-place
// restore must refuse before it changes anything. Backups without a recorded
// UID are told apart by the datastore schema.
func TestReconcileKubernetesRestoreRefusesRecreatedCluster(t *testing.T) {
for _, tc := range []struct {
name string
edit func(map[string]string)
}{
{"recorded UID", func(md map[string]string) { md[kubernetesControlPlaneUIDMetaKey] = "uid-of-the-deleted-cluster" }},
{"schema of a backup without the UID", func(md map[string]string) {
delete(md, kubernetesControlPlaneUIDMetaKey)
md[kubernetesDatastorePrefixMetaKey] = "uid-of-the-deleted-cluster"
}},
} {
t.Run(tc.name, func(t *testing.T) {
backup := k8sBackupForRestore()
tc.edit(backup.Spec.DriverMetadata)
run := newK8sRestoreRun(t, "", backup)
run.verifyAndCapture(true)
got := run.get()
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "deleted and created again") {
t.Fatalf("restore into a re-created cluster not refused: %+v", got.Status)
}
if kubernetesStateFromRestoreJob(got) != nil {
t.Fatal("restore mode recorded for a refused restore")
}
if run.suspended(k8sTestRelease) {
t.Fatal("cluster HelmRelease suspended by a refused restore")
}
if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 {
t.Fatalf("worker pool scaled to %d by a refused restore", n)
}
})
}
}
// TestFailedKubernetesRestoreResumesWithoutClusterAPI covers Cluster API going
// unserved after the workers were stopped: the pools cannot be read back, but
// the autoscaler and the suspended HelmReleases must still be put back.
func TestFailedKubernetesRestoreResumesWithoutClusterAPI(t *testing.T) {
run := newK8sRestoreRun(t, "", k8sBackupForRestore())
run.addMachine("w-1")
run.verifyAndCapture(true)
run.step() // workers stopping, the Machine still there
if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 {
t.Fatalf("autoscaler not stopped: %d", n)
}
for _, res := range []string{"machinedeployments", "machines"} {
run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", res, func(clienttesting.Action) (bool, runtime.Object, error) {
return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: "MachineDeployment"}, SearchedVersions: []string{"v1beta1"}}
})
}
run.step()
if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
t.Fatalf("restore not failed: %+v", got.Status)
}
if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 1 {
t.Fatalf("autoscaler left at %d replicas", n)
}
if run.suspended(k8sTestRelease) {
t.Fatal("cluster HelmRelease left suspended")
}
}
// TestRenderKubernetesTemplateRefusesUnexpectedSchema pins the allowlist every
// strategy Job is rendered through: the schema becomes the prefix the restore
// deletes in the etcd the whole tenant shares.
func TestRenderKubernetesTemplateRefusesUnexpectedSchema(t *testing.T) {
tmpl := corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "etcd", Args: []string{"{{ .Datastore.Prefix }} {{ .Datastore.SourcePrefix }}"}}}}}
render := func(prefix, source string) error {
_, err := renderKubernetesTemplate(tmpl, k8sTestApp, k8sTestNS, kubernetesModeRestore, "key", kubernetesDatastore{Prefix: prefix, SourcePrefix: source}, nil)
return err
}
for _, ok := range []string{"0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10", "tenant_root_kubernetes_bk1"} {
if err := render(ok, ok); err != nil {
t.Errorf("schema %q refused: %v", ok, err)
}
}
if err := render("tenant_a_kubernetes_c1", ""); err != nil {
t.Errorf("a Job without a source schema refused: %v", err)
}
for _, bad := range []string{"", "a/b", "/", "..", "a..b", "a.b", "Tenant_a", "a b", "a\nb", "a*"} {
if render(bad, "tenant_a_kubernetes_c1") == nil {
t.Errorf("schema %q accepted", bad)
}
if bad != "" && render("tenant_a_kubernetes_c1", bad) == nil {
t.Errorf("source schema %q accepted", bad)
}
}
}
// TestReconcileKubernetesRestoreRefusesEditedPrefix covers a Backup whose
// recorded prefix was edited: the restore fails in its preflight, before any
// Job runs against the shared etcd or the cluster is stopped.
func TestReconcileKubernetesRestoreRefusesEditedPrefix(t *testing.T) {
backup := k8sBackupForRestore()
backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] = "tenant_a_kubernetes_c1/.."
run := newK8sRestoreRun(t, "", backup)
run.step() // StartedAt
run.step() // preflight
got := run.get()
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") {
t.Fatalf("restore from an edited prefix not refused: %+v", got.Status)
}
jobs := &batchv1.JobList{}
if err := run.env.client.List(context.Background(), jobs); err != nil || len(jobs.Items) != 0 {
t.Fatalf("Jobs created for an edited prefix: %d %v", len(jobs.Items), err)
}
if run.suspended(k8sTestRelease) {
t.Fatal("cluster HelmRelease suspended")
}
}
// TestReconcileKubernetesBackupRefusesUnexpectedSchema covers a live
// TenantControlPlane reporting a schema outside the allowlist.
func TestReconcileKubernetesBackupRefusesUnexpectedSchema(t *testing.T) {
ctx := context.Background()
bj := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"},
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp,
}},
}
env := newKubernetesTestEnv(t, bj)
tcp, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(ctx, k8sTestRelease, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedField(tcp.Object, "tenant_a/kubernetes_c1", "status", "storage", "setup", "schema"); err != nil {
t.Fatal(err)
}
if _, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(ctx, tcp, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
get := func() *backupsv1alpha1.BackupJob {
out := &backupsv1alpha1.BackupJob{}
if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil {
t.Fatal(err)
}
return out
}
for i := 0; i < 4 && get().Status.Phase != backupsv1alpha1.BackupJobPhaseFailed; i++ {
if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil {
t.Fatal(err)
}
}
if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") {
t.Fatalf("backup of an unexpected schema not refused: %+v", got.Status)
}
jobs := &batchv1.JobList{}
if err := env.client.List(ctx, jobs); err != nil || len(jobs.Items) != 0 {
t.Fatalf("dump Job created for an unexpected schema: %d %v", len(jobs.Items), err)
}
}