From 0f44cf9e25d85c43c0909a02ba218e5142328c39 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 15 Oct 2026 15:18:17 +0400 Subject: [PATCH] feat(backups): add Kubernetes backup strategy for tenant clusters A tenant Kubernetes application had no backup route in cozy-default, and none of the existing strategies fits it. Its API state is a key prefix in the etcd DataStore the whole tenant shares, so the Etcd strategy's snapshot would roll every tenant cluster back at once; its control-plane CA, service account key and Talos secrets cannot be regenerated without breaking every node and credential issued under them; and its PersistentVolumes are DataVolumes in the tenant namespace that the application's own deletion removes. The new Kubernetes strategy backs up the cluster's etcd prefix with a Job that authenticates with the cluster's own datastore certificate, and the PKI Secrets and data DataVolumes with Velero. The HelmReleases of the cluster and its node pools are recorded on the Backup rather than in Velero, because a label selector for them would also pick up the worker system disks. A backup warns when a CA would expire before the backup does. A restore rebuilds a deleted cluster under its old CA, or rolls a live one back after stopping its control plane and workers, and lets the etcd prefix and the volumes be restored separately. Signed-off-by: Andrey Kolkov Assisted-by: LLM --- .../strategy/v1alpha1/kubernetes_types.go | 95 ++ .../v1alpha1/zz_generated.deepcopy.go | 103 ++ examples/backups/kubernetes/00-helpers.sh | 94 ++ .../backups/kubernetes/01-create-cluster.sh | 102 ++ .../backups/kubernetes/02-create-backupjob.sh | 36 + .../backups/kubernetes/03-restore-in-place.sh | 44 + .../kubernetes/04-restore-after-delete.sh | 39 + examples/backups/kubernetes/README.md | 68 + examples/backups/kubernetes/cleanup.sh | 18 + examples/backups/kubernetes/run-all.sh | 29 + .../backupcontroller/backup_controller.go | 3 + .../backupcontroller/backupjob_controller.go | 3 + .../backupjob_controller_test.go | 1 + .../backupcontroller/default_objects_gate.go | 13 +- .../default_objects_gate_test.go | 24 +- .../kubernetesstrategy_controller.go | 862 ++++++++++ .../kubernetesstrategy_controller_test.go | 1483 +++++++++++++++++ .../kubernetesstrategy_restore.go | 1129 +++++++++++++ .../backupcontroller/restorejob_controller.go | 8 + .../system/backupstrategy-controller/Makefile | 15 +- ...ategy.backups.cozystack.io_kubernetes.yaml | 147 ++ .../images/kubernetes-backup/Dockerfile | 14 + .../templates/backupclass-default.yaml | 7 + .../templates/rbac.yaml | 20 +- .../strategy-kubernetes-default.yaml | 304 ++++ .../tests/rendering_test.yaml | 36 +- .../tests/restore_rbac_test.yaml | 4 +- .../backupstrategy-controller/values.yaml | 18 + 28 files changed, 4697 insertions(+), 22 deletions(-) create mode 100644 api/backups/strategy/v1alpha1/kubernetes_types.go create mode 100755 examples/backups/kubernetes/00-helpers.sh create mode 100755 examples/backups/kubernetes/01-create-cluster.sh create mode 100755 examples/backups/kubernetes/02-create-backupjob.sh create mode 100755 examples/backups/kubernetes/03-restore-in-place.sh create mode 100755 examples/backups/kubernetes/04-restore-after-delete.sh create mode 100644 examples/backups/kubernetes/README.md create mode 100755 examples/backups/kubernetes/cleanup.sh create mode 100755 examples/backups/kubernetes/run-all.sh create mode 100644 internal/backupcontroller/kubernetesstrategy_controller.go create mode 100644 internal/backupcontroller/kubernetesstrategy_controller_test.go create mode 100644 internal/backupcontroller/kubernetesstrategy_restore.go create mode 100644 packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml create mode 100644 packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile create mode 100644 packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml diff --git a/api/backups/strategy/v1alpha1/kubernetes_types.go b/api/backups/strategy/v1alpha1/kubernetes_types.go new file mode 100644 index 000000000..704fc5d9f --- /dev/null +++ b/api/backups/strategy/v1alpha1/kubernetes_types.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// Package v1alpha1 defines strategy.backups.cozystack.io API types. +// +// Group: strategy.backups.cozystack.io +// Version: v1alpha1 +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, + &Kubernetes{}, + &KubernetesList{}, + ) + return nil + }) +} + +const ( + KubernetesStrategyKind = "Kubernetes" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster + +// Kubernetes defines a backup strategy for a tenant Kubernetes application. +// A tenant cluster keeps its state in two places the platform can reach: the +// cluster's key prefix in the tenant's shared etcd DataStore, and host-side +// objects (control-plane PKI Secrets, the application HelmReleases, and the +// DataVolumes that back the tenant's PersistentVolumes). The strategy captures +// the first with a one-shot Job and the second with Velero. The two are taken +// one after another, so a backup is crash-consistent at best. +type Kubernetes struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KubernetesSpec `json:"spec,omitempty"` + Status KubernetesStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// KubernetesList contains a list of Kubernetes backup strategies. +type KubernetesList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Kubernetes `json:"items"` +} + +// KubernetesSpec specifies the desired behavior of a Kubernetes backup strategy. +type KubernetesSpec struct { + // Template holds a PodTemplateSpec for the Job that dumps the cluster's + // key prefix from its etcd DataStore to object storage, writes it back on + // restore, and deletes the object on cleanup. The Job runs in the + // application namespace. Helm-like Go templates are supported in every + // string field and are rendered against: + // - `.Release.Name` and `.Release.Namespace` — the application's name and + // namespace; `.Release.Prefix` is `kubernetes-`. + // - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`. + // - `.ObjectKey` — the object-storage key of this backup's dump. + // - `.Datastore.Prefix` — the cluster's key prefix, without slashes. + // - `.Datastore.Endpoints` — comma-separated etcd client URLs. + // - `.Datastore.CertificateSecret` — the Secret holding the cluster's + // etcd client certificate (`ca.crt`, `server.crt`, `server.key`). + // - `.Parameters` — the resolved BackupClass strategy parameters. + // + // Stored schemaless for the same CRD-size reason as redis_types.go. + // + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Template corev1.PodTemplateSpec `json:"template"` + + // Velero holds the Velero Backup and Restore specs for the host-side + // objects and volumes. Rendered against `.Application` and `.Parameters`, + // like the Velero strategy. + // + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Velero VeleroTemplate `json:"velero"` + + // CertificateWarningThreshold is how close to expiry a control-plane + // certificate may be before a backup records a warning. Defaults to 720h. + // +optional + CertificateWarningThreshold *metav1.Duration `json:"certificateWarningThreshold,omitempty"` +} + +type KubernetesStatus struct { + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go b/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go index 3ad54f0fe..9efd1fb9b 100644 --- a/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go @@ -839,6 +839,109 @@ func (in *KafkaStatus) DeepCopy() *KafkaStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kubernetes. +func (in *Kubernetes) DeepCopy() *Kubernetes { + if in == nil { + return nil + } + out := new(Kubernetes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Kubernetes) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesList) DeepCopyInto(out *KubernetesList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Kubernetes, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesList. +func (in *KubernetesList) DeepCopy() *KubernetesList { + if in == nil { + return nil + } + out := new(KubernetesList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KubernetesList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + in.Velero.DeepCopyInto(&out.Velero) + if in.CertificateWarningThreshold != nil { + in, out := &in.CertificateWarningThreshold, &out.CertificateWarningThreshold + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec. +func (in *KubernetesSpec) DeepCopy() *KubernetesSpec { + if in == nil { + return nil + } + out := new(KubernetesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus. +func (in *KubernetesStatus) DeepCopy() *KubernetesStatus { + if in == nil { + return nil + } + out := new(KubernetesStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MariaDB) DeepCopyInto(out *MariaDB) { *out = *in diff --git a/examples/backups/kubernetes/00-helpers.sh b/examples/backups/kubernetes/00-helpers.sh new file mode 100755 index 000000000..401b5edb8 --- /dev/null +++ b/examples/backups/kubernetes/00-helpers.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Shared helpers for the tenant Kubernetes backup/restore walkthrough. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export NC='\033[0m' +export BOLD='\033[1m' + +export NAMESPACE="${NAMESPACE:-tenant-root}" +export CLUSTER_NAME="${CLUSTER_NAME:-backup-demo}" +export POOL_NAME="${POOL_NAME:-md0}" +export WORKER_DISK_SIZE="${WORKER_DISK_SIZE:-20Gi}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-cozy-default}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kubernetes-backup-job}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kubernetes-restore-inplace}" +export RESTOREJOB_RECREATE_NAME="${RESTOREJOB_RECREATE_NAME:-kubernetes-restore-recreate}" +export TENANT_API_PORT="${TENANT_API_PORT:-16443}" +export MARKER_VALUE="${MARKER_VALUE:-roundtrip-7c2e91d4}" +export RELEASE="kubernetes-${CLUSTER_NAME}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# The admin kubeconfig points at the in-cluster Service, so the walkthrough +# reaches the tenant API through a port-forward to it and keeps the TLS server +# name the certificate is issued for. +TENANT_KUBECONFIG="$(dirname "${BASH_SOURCE[0]}")/.tenant-kubeconfig" +export TENANT_KUBECONFIG + +tenant_connect() { + kubectl -n "$NAMESPACE" get secret "${RELEASE}-admin-kubeconfig" \ + -o jsonpath='{.data.super-admin\.svc}' | base64 -d > "$TENANT_KUBECONFIG" + chmod 600 "$TENANT_KUBECONFIG" + local cluster + cluster=$(kubectl --kubeconfig "$TENANT_KUBECONFIG" config view -o jsonpath='{.clusters[0].name}') + kubectl --kubeconfig "$TENANT_KUBECONFIG" config set-cluster "$cluster" \ + --server="https://127.0.0.1:${TENANT_API_PORT}" \ + --tls-server-name="${RELEASE}.${NAMESPACE}.svc" >/dev/null + pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true + kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}" "${TENANT_API_PORT}:6443" >/dev/null 2>&1 & + for _ in $(seq 1 30); do + tkubectl get --raw /readyz >/dev/null 2>&1 && return 0 + sleep 2 + done + log_error "tenant API of ${RELEASE} is not reachable through the port-forward" + return 1 +} + +tenant_disconnect() { + pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true +} + +tkubectl() { + kubectl --kubeconfig "$TENANT_KUBECONFIG" "$@" +} + +# The CA fingerprint is what proves a restore brought back the cluster rather +# than a new one: nodes, kubeconfigs and tokens all trust this CA. +cluster_ca_fingerprint() { + kubectl -n "$NAMESPACE" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' \ + | base64 -d | openssl x509 -noout -fingerprint -sha256 | cut -d= -f2 +} + +# wait_tenant_marker checks both halves of the backup: the ConfigMap comes back +# with etcd, the file on the PVC with the volumes. +wait_tenant_marker() { + local timeout="${1:-900}" elapsed=0 cm data + while true; do + cm=$(tkubectl -n demo get configmap marker -o jsonpath='{.data.marker}' 2>/dev/null || true) + data=$(tkubectl -n demo exec deploy/writer -- cat /data/marker 2>/dev/null || true) + if [[ "$cm" == "$MARKER_VALUE" && "$data" == "$MARKER_VALUE" ]]; then + log_success "marker present in etcd (ConfigMap) and on the volume" + return 0 + fi + (( elapsed >= timeout )) && { log_error "marker not restored: configmap='${cm}' volume='${data}'"; return 1; } + sleep 10 + elapsed=$((elapsed + 10)) + done +} diff --git a/examples/backups/kubernetes/01-create-cluster.sh b/examples/backups/kubernetes/01-create-cluster.sh new file mode 100755 index 000000000..8577ec202 --- /dev/null +++ b/examples/backups/kubernetes/01-create-cluster.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Step 01: Create a tenant Kubernetes cluster with one worker pool and seed it +# with a marker held in etcd (a ConfigMap) and on a PersistentVolume. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 01: Create Kubernetes '${CLUSTER_NAME}' and seed a marker" + +kubectl apply -f - </dev/null 2>&1 \ + && [[ -n "$(tkubectl get nodes -o name)" ]] \ + && [[ -n "$(tkubectl get storageclass -o name)" ]]; then + kubectl_ok=1 + break + fi + sleep 15 +done +(( kubectl_ok )) || { log_error "worker node or StorageClass did not appear"; exit 1; } + +tkubectl apply -f - < /data/marker; sync; sleep infinity"] + volumeMounts: + - {name: data, mountPath: /data} + volumes: + - name: data + persistentVolumeClaim: {claimName: data} +EOF + +wait_tenant_marker 900 +cluster_ca_fingerprint > "$SCRIPT_DIR/.ca-fingerprint" +log_success "cluster CA $(cat "$SCRIPT_DIR/.ca-fingerprint")" +tenant_disconnect + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupjob.sh" diff --git a/examples/backups/kubernetes/02-create-backupjob.sh b/examples/backups/kubernetes/02-create-backupjob.sh new file mode 100755 index 000000000..c290711c1 --- /dev/null +++ b/examples/backups/kubernetes/02-create-backupjob.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Step 02: Back the cluster up through the cozy-default BackupClass. The +# controller dumps the cluster's etcd prefix to the platform bucket with a +# Job, then backs up its control-plane Secrets and volumes with Velero. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 02: Create BackupJob '${BACKUPJOB_NAME}'" + +kubectl apply -f - </dev/null || true) +kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found +[[ -n "$backup_ref" ]] && kubectl -n "$NAMESPACE" delete backup "$backup_ref" --ignore-not-found +kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found +kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --ignore-not-found +rm -f "$TENANT_KUBECONFIG" "$SCRIPT_DIR/.ca-fingerprint" +log_success "done" diff --git a/examples/backups/kubernetes/run-all.sh b/examples/backups/kubernetes/run-all.sh new file mode 100755 index 000000000..731448b4d --- /dev/null +++ b/examples/backups/kubernetes/run-all.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# End-to-end tenant Kubernetes backup/restore walkthrough. It proves the +# restore brings back the same cluster: the CA fingerprint is unchanged, and a +# marker kept in etcd and one kept on a PersistentVolume both come back, first +# after rolling a live cluster back, then after rebuilding a deleted one. +# +# Needs the platform default backups (backupstrategy-controller with a Velero +# BSL and the system bucket), so cozy-default routes Kubernetes to +# cozy-default-kubernetes. +# +# Env knobs: see 00-helpers.sh. SKIP_RESTORE=1 stops after the backup. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +kubectl wait --for=create kubernetes.strategy.backups.cozystack.io/cozy-default-kubernetes --timeout=15m >/dev/null \ + || { log_error "cozy-default-kubernetes strategy is missing: enable the platform default backups first"; exit 1; } + +steps=(01 02 03 04) +[[ "${SKIP_RESTORE:-0}" == "1" ]] && steps=(01 02) +for step in "${steps[@]}"; do + script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)" + [[ -n "$script" ]] || { echo "no script found for step $step" >&2; exit 1; } + echo "=== Running $(basename "$script") ===" + bash "$script" +done + +print_header "Walkthrough complete" diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..6741fbaf6 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -183,6 +183,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // deletes the object and WAITS for that delete before the Backup is // removed, so no object is orphaned (see cleanupKafkaBackup). return r.cleanupKafkaBackup(ctx, backup) + case strategyv1alpha1.KubernetesStrategyKind: + // Owns two artifacts: the Velero backup and the etcd dump object. + return r.cleanupKubernetesBackup(ctx, backup) case strategyv1alpha1.VeleroStrategyKind: return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup) default: diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index b86278649..256a9aaf5 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -214,6 +214,8 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return r.reconcileRedis(ctx, j, resolved) case strategyv1alpha1.KafkaStrategyKind: return r.reconcileKafka(ctx, j, resolved) + case strategyv1alpha1.KubernetesStrategyKind: + return r.reconcileKubernetes(ctx, j, resolved) default: logger.V(1).Info("BackupJob resolved StrategyRef.Kind not supported, skipping", "backupjob", j.Name, @@ -240,6 +242,7 @@ func supportedBackupStrategyKinds() []string { strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind, + strategyv1alpha1.KubernetesStrategyKind, } } diff --git a/internal/backupcontroller/backupjob_controller_test.go b/internal/backupcontroller/backupjob_controller_test.go index f802fe374..994654987 100644 --- a/internal/backupcontroller/backupjob_controller_test.go +++ b/internal/backupcontroller/backupjob_controller_test.go @@ -45,6 +45,7 @@ func TestSupportedBackupStrategyKindsMatchesDispatch(t *testing.T) { strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind, + strategyv1alpha1.KubernetesStrategyKind, } sort.Strings(got) sort.Strings(want) diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 9aea96f31..94075651e 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -593,12 +593,13 @@ func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *ba group = *strategy.StrategyRef.APIGroup } // When the Velero BSL is disabled (velero.bslEnabled=false, surfaced - // here as an empty VeleroNamespace) the chart gates the Velero - // Strategy CRs off the SAME flag, so they are never rendered. The - // BackupClass still routes to them unconditionally, so counting them - // as missing would force a Helm upgrade every MinForceInterval - // forever against a render that can never produce them. - if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" { + // here as an empty VeleroNamespace) the chart gates the Strategy CRs + // that back up through Velero (Velero, Kubernetes) off the SAME flag, + // so they are never rendered. The BackupClass still routes to them + // unconditionally, so counting them as missing would force a Helm + // upgrade every MinForceInterval forever against a render that can + // never produce them. + if g.VeleroNamespace == "" && group == strategyAPIGroup && (kind == "Velero" || kind == "Kubernetes") { continue } key := fmt.Sprintf("%s/%s", kind, name) diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index e287c9872..a07beea7f 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -37,6 +37,7 @@ var gateGVKs = []schema.GroupVersionKind{ {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Kubernetes"}, {Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"}, {Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, } @@ -109,6 +110,7 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { ref("Altinity", "cozy-default-altinity"), ref("Velero", "cozy-default-velero-vminstance"), ref("Velero", "cozy-default-velero-vmdisk"), + ref("Kubernetes", "cozy-default-kubernetes"), }, }, } @@ -149,6 +151,7 @@ func allDefaultObjects() []runtime.Object { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), } } @@ -288,8 +291,8 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) { t.Fatal("expected a forced Helm upgrade when default objects are missing") } // 6 strategyRefs (Velero twice under different names) + the BSL. - if len(missing) != 7 { - t.Fatalf("missing = %v, want 7 entries", missing) + if len(missing) != 8 { + t.Fatalf("missing = %v, want 8 entries", missing) } ann := forceAnnotations(t, dyn) forceAt, ok := ann["reconcile.fluxcd.io/forceAt"] @@ -319,6 +322,7 @@ func TestCheckNoopWhenAllPresent(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), ) @@ -454,8 +458,8 @@ func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) { if !forced { t.Fatal("objects release throttled by the earlier credentials force") } - if len(missing) != 7 { - t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing) + if len(missing) != 8 { + t.Fatalf("missing = %v, want the 7 strategies + the BSL", missing) } if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { t.Errorf("own HelmRelease not forced: %v", ann) @@ -482,7 +486,7 @@ func TestForceSkipsSuspendedHelmRelease(t *testing.T) { if forced { t.Error("forced reported true for a suspended HelmRelease that ignores the annotations") } - if len(missing) != 7 { + if len(missing) != 8 { t.Fatalf("missing = %v, want the objects still reported", missing) } if ann := forceAnnotations(t, dyn); len(ann) != 0 { @@ -599,6 +603,7 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), ) @@ -658,6 +663,7 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), ) // Rebuild the RESTMapper without BackupStorageLocation: the velero.io API // is not served on this cluster. @@ -1053,8 +1059,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { g.lastForce = base.Add(-time.Minute) g.now = func() time.Time { return base } - if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 { - t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err) + if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != len(allDefaultObjects()) { + t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want %d reported, none forced", missing, forced, err, len(allDefaultObjects())) } g.now = func() time.Time { return base.Add(5 * time.Minute) } @@ -1065,8 +1071,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { g.now = func() time.Time { return base.Add(time.Hour) } missing, forced, err := g.Check(context.Background()) - if err != nil || !forced || len(missing) != 7 { - t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err) + if err != nil || !forced || len(missing) != len(allDefaultObjects()) { + t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want %d reported and forced again", missing, forced, err, len(allDefaultObjects())) } if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first { t.Errorf("not forced again for unannotated objects: forceAt still %q", first) diff --git a/internal/backupcontroller/kubernetesstrategy_controller.go b/internal/backupcontroller/kubernetesstrategy_controller.go new file mode 100644 index 000000000..1fd47f195 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_controller.go @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "maps" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/template" +) + +const ( + kubernetesAppKind = "Kubernetes" + kubernetesReleasePrefix = "kubernetes-" + + kubernetesLabelMode = "kubernetes.strategy.backups.cozystack.io/mode" + kubernetesModeBackup = "backup" + kubernetesModeRestore = "restore" + kubernetesModeCleanup = "cleanup" + // kubernetesModeVerify checks that the dump object exists before a restore + // changes anything. + kubernetesModeVerify = "verify" + + // kubernetesCleanupAttemptsAnnotation counts failed cleanup Jobs on the + // Backup, so a persistently failing delete (revoked credentials, a removed + // bucket) ends in a released Backup and a Warning rather than a Job storm. + kubernetesCleanupAttemptsAnnotation = "kubernetes.strategy.backups.cozystack.io/cleanup-attempts" + kubernetesCleanupMaxAttempts = 5 + + kubernetesMetaPrefix = "kubernetes.strategy.backups.cozystack.io/" + kubernetesObjectMetaKey = kubernetesMetaPrefix + "object" + kubernetesDatastorePrefixMetaKey = kubernetesMetaPrefix + "datastore-prefix" + kubernetesDatastoreEndpointsMetaKey = kubernetesMetaPrefix + "datastore-endpoints" + kubernetesHelmReleasesMetaKey = kubernetesMetaPrefix + "helmreleases" + kubernetesDataVolumesMetaKey = kubernetesMetaPrefix + "datavolumes" + kubernetesCertWarningsMetaKey = kubernetesMetaPrefix + "certificate-warnings" + kubernetesControlPlaneUIDMetaKey = kubernetesMetaPrefix + "tenantcontrolplane-uid" + kubernetesParamPrefix = kubernetesMetaPrefix + "parameter/" + + // kubernetesCondCertificates is set on the BackupJob once the control-plane + // certificates have been checked; Status=True means at least one expires + // inside the warning window. + kubernetesCondCertificates = "CertificatesExpiring" + + kubernetesPollInterval = 5 * time.Second + // Leaf certificates are rotated by Kamaji and cert-manager, so a near + // expiry is only worth a warning when it is this close. + kubernetesDefaultCertThreshold = 720 * time.Hour + // A CA must outlive every backup that may still be restored, so its + // window is the Velero backup TTL; this is the fallback when none is set. + kubernetesDefaultBackupTTL = 720 * time.Hour + + helmReleaseNameAnnotation = "meta.helm.sh/release-name" + clusterNameLabel = "cluster.x-k8s.io/cluster-name" + helmInstanceLabel = "app.kubernetes.io/instance" + kamajiNameLabel = "kamaji.clastix.io/name" +) + +var ( + tenantControlPlaneGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "tenantcontrolplanes"} + kamajiDataStoreGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "datastores"} + kamajiControlPlaneGVR = schema.GroupVersionResource{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Resource: "kamajicontrolplanes"} + machineDeploymentGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"} + machineGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machines"} + deploymentGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} +) + +// kubernetesCertificateSecrets lists, by suffix after the release name, the +// Secrets whose certificates a backup checks. Kamaji and cert-manager rotate +// the leaves; the CAs are irreplaceable, so a restore needs them valid. +var kubernetesCertificateSecrets = []string{ + "ca", + "front-proxy-ca-certificate", + "api-server-certificate", + "api-server-kubelet-client-certificate", + "front-proxy-client-certificate", + "talos-ca", + "talos-tls-cert", +} + +// kubernetesUnlabelledSecrets are issued by cert-manager without the Helm +// instance label the Velero selector relies on; the driver adds it before the +// backup so the Talos PKI is not silently left out. +var kubernetesUnlabelledSecrets = []string{"talos-ca", "talos-tls-cert"} + +func kubernetesReleaseName(appName string) string { + return kubernetesReleasePrefix + appName +} + +func validateKubernetesApplicationRef(ref corev1.TypedLocalObjectReference) error { + if ref.Kind != kubernetesAppKind { + return fmt.Errorf("kubernetes strategy supports applicationRef.kind %s, got %q", kubernetesAppKind, ref.Kind) + } + if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup { + return fmt.Errorf("kubernetes strategy supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) + } + return nil +} + +func kubernetesObjectKey(namespace, appName, backupName string) string { + return fmt.Sprintf("%s/%s/%s.etcd.json", namespace, appName, backupName) +} + +// kubernetesDatastore is where the tenant cluster keeps its state: a key +// prefix in the etcd DataStore shared by every cluster of the tenant. +type kubernetesDatastore struct { + // ControlPlaneUID identifies the cluster across a delete and re-create + // under the same name, which Prefix does not on a Kamaji that keys the + // schema on the namespace and name. + ControlPlaneUID string + Prefix string + Endpoints []string + CertificateSecret string + // SourcePrefix is the prefix the restored dump was taken under. It + // differs from Prefix when a deleted cluster is rebuilt: Kamaji keys a new + // TenantControlPlane's schema on its UID. + SourcePrefix string +} + +func (d kubernetesDatastore) templateContext() map[string]any { + source := d.SourcePrefix + if source == "" { + source = d.Prefix + } + return map[string]any{ + "Prefix": d.Prefix, + "SourcePrefix": source, + "Endpoints": strings.Join(d.Endpoints, ","), + "CertificateSecret": d.CertificateSecret, + } +} + +// etcdEndpointURL gives a scheme to a DataStore endpoint: Kamaji lists them as +// host:port and always talks TLS to an etcd DataStore. +func etcdEndpointURL(endpoint string) string { + if strings.Contains(endpoint, "://") { + return endpoint + } + return "https://" + endpoint +} + +// resolveKubernetesDatastore reads the cluster's key prefix and client +// certificate from its TenantControlPlane and the client URLs from the +// DataStore it points at. +func resolveKubernetesDatastore(ctx context.Context, dyn dynamic.Interface, namespace, appName string) (*kubernetesDatastore, error) { + release := kubernetesReleaseName(appName) + tcp, err := dyn.Resource(tenantControlPlaneGVR).Namespace(namespace).Get(ctx, release, metav1.GetOptions{}) + if err != nil { + return nil, err + } + if driver, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "driver"); driver != "" && driver != "etcd" { + return nil, fmt.Errorf("TenantControlPlane %s/%s uses the %q datastore driver; only etcd is supported", namespace, release, driver) + } + prefix, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + if prefix == "" { + prefix, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStoreSchema") + } + if prefix == "" { + return nil, fmt.Errorf("TenantControlPlane %s/%s reports no datastore schema yet", namespace, release) + } + dsName, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "dataStoreName") + if dsName == "" { + dsName, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStore") + } + if dsName == "" { + return nil, fmt.Errorf("TenantControlPlane %s/%s names no DataStore", namespace, release) + } + certSecret, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "certificate", "secretName") + if certSecret == "" { + certSecret = release + "-datastore-certificate" + } + ds, err := dyn.Resource(kamajiDataStoreGVR).Get(ctx, dsName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get DataStore %s: %w", dsName, err) + } + raw, _, _ := unstructured.NestedStringSlice(ds.Object, "spec", "endpoints") + if len(raw) == 0 { + return nil, fmt.Errorf("DataStore %s lists no endpoints", dsName) + } + endpoints := make([]string, 0, len(raw)) + for _, e := range raw { + endpoints = append(endpoints, etcdEndpointURL(e)) + } + return &kubernetesDatastore{ControlPlaneUID: string(tcp.GetUID()), Prefix: prefix, Endpoints: endpoints, CertificateSecret: certSecret}, nil +} + +// kubernetesCertificateWarnings reports every certificate in the given Secrets +// that expires inside its window: caWindow for CAs, leafWindow for the rest. +// Output is sorted so a condition message is stable across reconciles. +func kubernetesCertificateWarnings(secrets []corev1.Secret, now time.Time, caWindow, leafWindow time.Duration) []string { + seen := map[string]bool{} + var out []string + for _, s := range secrets { + for key, data := range s.Data { + if !strings.HasSuffix(key, ".crt") { + continue + } + for rest := data; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + window := leafWindow + kind := "certificate" + if cert.IsCA { + window = caWindow + kind = "CA" + } + if !cert.NotAfter.Before(now.Add(window)) { + continue + } + msg := fmt.Sprintf("%s %s/%s (CN=%s) expires %s", kind, s.Name, key, cert.Subject.CommonName, cert.NotAfter.UTC().Format(time.RFC3339)) + if !seen[msg] { + seen[msg] = true + out = append(out, msg) + } + } + } + } + sort.Strings(out) + return out +} + +// kubernetesHelmReleaseNames returns the HelmRelease of the cluster followed +// by those of its worker pools. Pools live in separate kubernetes-nodes +// releases; the MachineDeployments of the cluster name them, which also +// covers the older layout where the cluster release owned its pools. +func kubernetesHelmReleaseNames(mds []unstructured.Unstructured, release string) []string { + names := []string{release} + for _, md := range mds { + if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster != release { + continue + } + if owner := md.GetAnnotations()[helmReleaseNameAnnotation]; owner != "" && !slices.Contains(names, owner) { + names = append(names, owner) + } + } + return names +} + +// sanitizeHelmRelease keeps what is needed to recreate a HelmRelease and +// drops server-set metadata and status. +func sanitizeHelmRelease(hr *unstructured.Unstructured) map[string]any { + labels := map[string]string{} + for k, v := range hr.GetLabels() { + if !strings.HasPrefix(k, "velero.io/") { + labels[k] = v + } + } + meta := map[string]any{ + "name": hr.GetName(), + "namespace": hr.GetNamespace(), + } + if len(labels) > 0 { + meta["labels"] = labels + } + spec, _, _ := unstructured.NestedMap(hr.Object, "spec") + delete(spec, "suspend") + return map[string]any{ + "apiVersion": hr.GetAPIVersion(), + "kind": hr.GetKind(), + "metadata": meta, + "spec": spec, + } +} + +// kubernetesSchemaPattern is every datastore schema Kamaji assigns: a +// TenantControlPlane UID, or _ on an older Kamaji. The Job +// turns a schema into the prefix it runs `etcdctl del --prefix` on in the +// etcd every cluster of the tenant shares, so a value that is empty, holds a +// slash or a dot, or comes from an edited Backup must never reach it. +var kubernetesSchemaPattern = regexp.MustCompile(`^[a-z0-9_-]+$`) + +func validateKubernetesDatastore(ds kubernetesDatastore) error { + if !kubernetesSchemaPattern.MatchString(ds.Prefix) { + return fmt.Errorf("refusing datastore schema %q: want %s", ds.Prefix, kubernetesSchemaPattern) + } + if ds.SourcePrefix != "" && !kubernetesSchemaPattern.MatchString(ds.SourcePrefix) { + return fmt.Errorf("refusing source datastore schema %q: want %s", ds.SourcePrefix, kubernetesSchemaPattern) + } + return nil +} + +func renderKubernetesTemplate( + tmpl corev1.PodTemplateSpec, + appName, namespace, mode, objectKey string, + ds kubernetesDatastore, + parameters map[string]string, +) (*corev1.PodTemplateSpec, error) { + if err := validateKubernetesDatastore(ds); err != nil { + return nil, err + } + return template.Template(&tmpl, map[string]any{ + "Release": map[string]string{ + "Name": appName, + "Namespace": namespace, + "Prefix": kubernetesReleasePrefix, + }, + "Mode": mode, + "ObjectKey": objectKey, + "Datastore": ds.templateContext(), + "Parameters": parameters, + }) +} + +func kubernetesStrategyParameters(b *backupsv1alpha1.Backup) map[string]string { + out := map[string]string{} + for k, v := range b.Spec.DriverMetadata { + if p := strings.TrimPrefix(k, kubernetesParamPrefix); p != k && p != "" { + out[p] = v + } + } + return out +} + +func kubernetesBackupDatastore(b *backupsv1alpha1.Backup) kubernetesDatastore { + ds := kubernetesDatastore{ + Prefix: b.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey], + CertificateSecret: kubernetesReleaseName(b.Spec.ApplicationRef.Name) + "-datastore-certificate", + } + if e := b.Spec.DriverMetadata[kubernetesDatastoreEndpointsMetaKey]; e != "" { + ds.Endpoints = strings.Split(e, ",") + } + return ds +} + +func kubernetesBackupTTL(s *strategyv1alpha1.Kubernetes) time.Duration { + if s.Spec.Velero.Spec.TTL.Duration > 0 { + return s.Spec.Velero.Spec.TTL.Duration + } + return kubernetesDefaultBackupTTL +} + +func kubernetesCertThreshold(s *strategyv1alpha1.Kubernetes) time.Duration { + if s.Spec.CertificateWarningThreshold != nil && s.Spec.CertificateWarningThreshold.Duration > 0 { + return s.Spec.CertificateWarningThreshold.Duration + } + return kubernetesDefaultCertThreshold +} + +// --------------------------------------------------------------------------- +// BackupJob path +// --------------------------------------------------------------------------- + +// reconcileKubernetes drives a BackupJob for a tenant Kubernetes application in +// three steps, each observed on later reconciles: check the control-plane +// certificates, dump the cluster's etcd prefix with a Job, then back up the +// host-side Secrets and volumes with Velero. The Backup records what a restore +// needs to put both halves back. +func (r *BackupJobReconciler) reconcileKubernetes(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) { + logger := getLogger(ctx) + logger.Debug("reconciling Kubernetes strategy", "backupjob", j.Name, "phase", j.Status.Phase) + + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || + j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + return ctrl.Result{}, nil + } + if err := validateKubernetesApplicationRef(j.Spec.ApplicationRef); err != nil { + return r.markBackupJobFailed(ctx, j, err.Error()) + } + + if j.Status.StartedAt == nil { + fresh := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil { + return ctrl.Result{}, err + } + if fresh.Status.StartedAt == nil { + base := fresh.DeepCopy() + now := metav1.Now() + fresh.Status.StartedAt = &now + fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + j.Status.StartedAt = fresh.Status.StartedAt + j.Status.Phase = fresh.Status.Phase + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name) + } + return ctrl.Result{}, err + } + + app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef) + if err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application not found: %s/%s", j.Namespace, j.Spec.ApplicationRef.Name)) + } + return ctrl.Result{}, err + } + + appName := j.Spec.ApplicationRef.Name + release := kubernetesReleaseName(appName) + ds, err := resolveKubernetesDatastore(ctx, r.Interface, j.Namespace, appName) + if err != nil { + if apierrors.IsNotFound(err) { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s has no TenantControlPlane: %v", j.Namespace, appName, err)) + } + return r.markBackupJobFailed(ctx, j, err.Error()) + } + + // A restore that failed partway leaves the control plane stopped on a + // possibly partial prefix, marked with the replica count it had. Backing + // that up would turn the damage into a recovery point, so a run that has + // not dumped anything yet refuses. + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: jobNameForBackupJob(j)}, &batchv1.Job{}); apierrors.IsNotFound(err) { + kcp, kerr := r.Interface.Resource(kamajiControlPlaneGVR).Namespace(j.Namespace).Get(ctx, release, metav1.GetOptions{}) + if kerr != nil && !apierrors.IsNotFound(kerr) && !apimeta.IsNoMatchError(kerr) { + return ctrl.Result{}, kerr + } + if kerr == nil { + if _, stopped := kcp.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; stopped { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s is stopped by an unfinished restore; finish or repair the restore before backing it up", j.Namespace, appName)) + } + } + } else if err != nil { + return ctrl.Result{}, err + } + + if apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates) == nil { + return r.checkKubernetesCertificates(ctx, j, strategy, release) + } + + objectKey := kubernetesObjectKey(j.Namespace, appName, j.Name) + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, appName, j.Namespace, kubernetesModeBackup, objectKey, *ds, resolved.Parameters) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Kubernetes strategy: %v", err)) + } + dump, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, jobNameForBackupJob(j), + map[string]string{ + kubernetesLabelMode: kubernetesModeBackup, + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, rendered) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure etcd dump Job: %v", err)) + } + switch jobConditionState(dump) { + case batchv1.JobFailed: + msg := jobFailureMessage(dump) + if msg == "" { + msg = "etcd dump Job reported Failed" + } + return r.markBackupJobFailed(ctx, j, msg) + case batchv1.JobComplete: + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + veleroBackups := &velerov1.BackupList{} + if err := r.List(ctx, veleroBackups, client.InNamespace(veleroNamespace), client.MatchingLabels{ + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + backupsv1alpha1.OwningJobNameLabel: j.Name, + }); err != nil { + return ctrl.Result{}, err + } + if len(veleroBackups.Items) == 0 { + if err := r.labelKubernetesSecrets(ctx, j.Namespace, release); err != nil { + return ctrl.Result{}, err + } + if err := r.createKubernetesVeleroBackup(ctx, j, strategy, app, resolved); err != nil { + return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, fmt.Sprintf("failed to create Velero Backup: %v", err)) + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + if len(veleroBackups.Items) > 1 { + return r.markBackupJobFailed(ctx, j, "more than one Velero Backup references this BackupJob") + } + vb := &veleroBackups.Items[0] + switch vb.Status.Phase { + case velerov1.BackupPhaseCompleted: + case velerov1.BackupPhaseFailed, velerov1.BackupPhasePartiallyFailed, velerov1.BackupPhaseFailedValidation: + return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, vb)) + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + if j.Status.BackupRef != nil { + return ctrl.Result{}, nil + } + // A failed create is retried rather than failing the run: the dump and + // the Velero backup already exist, and no Backup would own them. + artifact, err := r.createKubernetesBackupArtifact(ctx, j, resolved, vb, *ds, objectKey) + if err != nil { + return ctrl.Result{}, fmt.Errorf("create Backup artifact: %w", err) + } + now := metav1.Now() + j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} + j.Status.CompletedAt = &now + j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "BackupCompleted", + Message: "etcd prefix and Velero backup completed", + }) + return ctrl.Result{}, r.Status().Update(ctx, j) +} + +// failKubernetesBackupAfterDump fails a run whose etcd dump is already in the +// bucket. No Backup will own the dump, so a cleanup-mode Job owned by the +// BackupJob removes it; the dump holds every Secret of the cluster. +func (r *BackupJobReconciler) failKubernetesBackupAfterDump(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, ds kubernetesDatastore, objectKey string, resolved *ResolvedBackupConfig, msg string) (ctrl.Result, error) { + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, j.Spec.ApplicationRef.Name, j.Namespace, kubernetesModeCleanup, objectKey, ds, resolved.Parameters) + if err == nil { + _, err = ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, j.Name+"-discard", + map[string]string{ + kubernetesLabelMode: kubernetesModeCleanup, + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, rendered) + } + if err != nil { + msg = fmt.Sprintf("%s; the etcd dump %s could not be scheduled for deletion: %v", msg, objectKey, err) + } + return r.markBackupJobFailed(ctx, j, msg) +} + +// checkKubernetesCertificates records on the BackupJob whether any +// control-plane certificate expires inside its window. It never fails the +// run: an expiring certificate is a reason to act, not to skip the backup. +func (r *BackupJobReconciler) checkKubernetesCertificates(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, release string) (ctrl.Result, error) { + var secrets []corev1.Secret + for _, suffix := range kubernetesCertificateSecrets { + s := corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: release + "-" + suffix}, &s) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return ctrl.Result{}, err + } + secrets = append(secrets, s) + } + warnings := kubernetesCertificateWarnings(secrets, time.Now(), kubernetesBackupTTL(strategy), kubernetesCertThreshold(strategy)) + cond := metav1.Condition{ + Type: kubernetesCondCertificates, + Status: metav1.ConditionFalse, + Reason: "CertificatesValid", + Message: "no control-plane certificate expires within the warning window", + } + if len(warnings) > 0 { + cond.Status = metav1.ConditionTrue + cond.Reason = "CertificatesExpiring" + cond.Message = strings.Join(warnings, "; ") + if r.Recorder != nil { + r.Recorder.Event(j, corev1.EventTypeWarning, "CertificatesExpiring", cond.Message) + } + } + apimeta.SetStatusCondition(&j.Status.Conditions, cond) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +func (r *BackupJobReconciler) labelKubernetesSecrets(ctx context.Context, namespace, release string) error { + for _, suffix := range kubernetesUnlabelledSecrets { + s := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release + "-" + suffix}, s) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return err + } + if s.Labels[helmInstanceLabel] == release { + continue + } + base := s.DeepCopy() + if s.Labels == nil { + s.Labels = map[string]string{} + } + s.Labels[helmInstanceLabel] = release + if err := r.Patch(ctx, s, client.MergeFrom(base)); err != nil { + return fmt.Errorf("label Secret %s/%s: %w", namespace, s.Name, err) + } + } + return nil +} + +func (r *BackupJobReconciler) createKubernetesVeleroBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, app map[string]any, resolved *ResolvedBackupConfig) error { + spec, err := template.Template(&strategy.Spec.Velero.Spec, map[string]any{ + "Application": app, + "Parameters": resolved.Parameters, + }) + if err != nil { + return err + } + // The volumes an in-place restore deletes and replaces are listed before + // Velero starts: each of them is then in the Velero backup unless it was + // deleted first, and a volume created while the backup runs, which Velero + // may have missed, is never deleted by a restore of it. + volumes, err := r.kubernetesDataVolumeNames(ctx, j.Namespace, kubernetesReleaseName(j.Spec.ApplicationRef.Name)) + if err != nil { + return err + } + vb := &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s.%s-", j.Namespace, j.Name), + Namespace: veleroNamespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, + Annotations: map[string]string{kubernetesDataVolumesMetaKey: strings.Join(volumes, ",")}, + }, + Spec: *spec, + } + if err := r.Create(ctx, vb); err != nil { + return err + } + if r.Recorder != nil { + r.Recorder.Event(j, corev1.EventTypeNormal, "VeleroBackupCreated", fmt.Sprintf("Created Velero Backup %s/%s", veleroNamespace, vb.Name)) + } + return nil +} + +// kubernetesDataVolumeNames lists the claims backing the cluster's +// PersistentVolumes: the DataVolumes and PVCs the CSI driver labels with the +// cluster name. A PVC is listed even without a DataVolume, so an in-place +// restore clears it too rather than leaving Velero to skip it. +func (r *BackupJobReconciler) kubernetesDataVolumeNames(ctx context.Context, ns, release string) ([]string, error) { + selector := clusterNameLabel + "=" + release + seen := map[string]bool{} + dvs, err := r.Interface.Resource(dataVolumeGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + for _, dv := range dvs.Items { + seen[dv.GetName()] = true + } + pvcs := &corev1.PersistentVolumeClaimList{} + if err := r.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{clusterNameLabel: release}); err != nil { + return nil, err + } + for _, pvc := range pvcs.Items { + seen[pvc.Name] = true + } + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// createKubernetesBackupArtifact records, besides the Velero backup and the +// etcd object, the cluster's HelmReleases and data DataVolumes. The +// HelmReleases are kept here rather than in Velero because selecting them by +// application label would also select the worker system disks, which the +// lineage webhook labels after the same application. +func (r *BackupJobReconciler) createKubernetesBackupArtifact( + ctx context.Context, + j *backupsv1alpha1.BackupJob, + resolved *ResolvedBackupConfig, + vb *velerov1.Backup, + ds kubernetesDatastore, + objectKey string, +) (*backupsv1alpha1.Backup, error) { + release := kubernetesReleaseName(j.Spec.ApplicationRef.Name) + mds, err := r.Interface.Resource(machineDeploymentGVR).Namespace(j.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil && !apierrors.IsNotFound(err) && !apimeta.IsNoMatchError(err) { + return nil, err + } + var mdItems []unstructured.Unstructured + if mds != nil { + mdItems = mds.Items + } + var releases []map[string]any + for _, name := range kubernetesHelmReleaseNames(mdItems, release) { + hr, err := r.Interface.Resource(helmReleaseGVR).Namespace(j.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + releases = append(releases, sanitizeHelmRelease(hr)) + } + releasesJSON, err := json.Marshal(releases) + if err != nil { + return nil, err + } + + driverMD := map[string]string{ + veleroBackupNameMetadataKey: vb.Name, + veleroBackupNamespaceMetadataKey: vb.Namespace, + kubernetesObjectMetaKey: objectKey, + kubernetesDatastorePrefixMetaKey: ds.Prefix, + kubernetesDatastoreEndpointsMetaKey: strings.Join(ds.Endpoints, ","), + kubernetesHelmReleasesMetaKey: string(releasesJSON), + kubernetesDataVolumesMetaKey: vb.Annotations[kubernetesDataVolumesMetaKey], + kubernetesControlPlaneUIDMetaKey: ds.ControlPlaneUID, + } + if c := apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates); c != nil && c.Status == metav1.ConditionTrue { + driverMD[kubernetesCertWarningsMetaKey] = c.Message + } + for k, v := range resolved.Parameters { + driverMD[kubernetesParamPrefix+k] = v + } + + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: j.Name, Namespace: j.Namespace}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: j.Spec.ApplicationRef, + StrategyRef: resolved.StrategyRef, + TakenAt: metav1.Now(), + DriverMetadata: driverMD, + PlanRef: j.Spec.PlanRef, + }, + Status: backupsv1alpha1.BackupStatus{ + Phase: backupsv1alpha1.BackupPhaseReady, + Artifact: &backupsv1alpha1.BackupArtifact{URI: fmt.Sprintf("velero://%s/%s", vb.Namespace, vb.Name)}, + }, + } + if err := r.Create(ctx, backup); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + existing := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); err != nil { + return nil, err + } + return existing, nil + } + return backup, nil +} + +// --------------------------------------------------------------------------- +// Backup deletion cleanup +// --------------------------------------------------------------------------- + +// cleanupKubernetesBackup removes both halves of a deleted Backup: the Velero +// backup through a DeleteBackupRequest, and the etcd dump object through a +// cleanup-mode Job, waiting for the Job before the Backup is released. The +// escape hatches match the Redis driver's. +func (r *BackupReconciler) cleanupKubernetesBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + if err := r.cleanupVeleroBackup(ctx, backup); err != nil { + return ctrl.Result{}, err + } + + jobName := backup.Name + "-cleanup" + if backup.Annotations[redisSkipArtifactCleanupAnnotation] == "true" { + existing := &batchv1.Job{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, existing); err == nil { + _ = r.deleteRedisCleanupJob(ctx, existing) + } + return ctrl.Result{}, nil + } + objectKey := backup.Spec.DriverMetadata[kubernetesObjectMetaKey] + if objectKey == "" { + return ctrl.Result{}, nil + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.releaseRedisCleanup(ctx, backup, objectKey, "strategy CR is gone"), nil + } + return ctrl.Result{}, err + } + + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job) + switch { + case apierrors.IsNotFound(err): + terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace) + if nsErr != nil { + return ctrl.Result{}, nsErr + } + if terminating { + return r.releaseRedisCleanup(ctx, backup, objectKey, "namespace is terminating"), nil + } + if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot project credentials: %v", perr)), nil + } + rendered, rerr := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, backup.Namespace, + kubernetesModeCleanup, objectKey, kubernetesBackupDatastore(backup), kubernetesStrategyParameters(backup)) + if rerr != nil { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cleanup template render failed: %v", rerr)), nil + } + cleanup := buildRedisCleanupJob(backup.Namespace, jobName, rendered) + cleanup.Labels = map[string]string{kubernetesLabelMode: kubernetesModeCleanup} + delete(cleanup.Spec.Template.Labels, redisLabelMode) + maps.Copy(cleanup.Spec.Template.Labels, cleanup.Labels) + if cerr := r.Create(ctx, cleanup); cerr != nil && !apierrors.IsAlreadyExists(cerr) { + if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil + } + return ctrl.Result{}, cerr + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + case err != nil: + return ctrl.Result{}, err + } + + if !job.DeletionTimestamp.IsZero() { + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + switch jobConditionState(job) { + case batchv1.JobComplete: + _ = r.deleteRedisCleanupJob(ctx, job) + return ctrl.Result{}, nil + case batchv1.JobFailed: + attempts, _ := strconv.Atoi(backup.Annotations[kubernetesCleanupAttemptsAnnotation]) + attempts++ + if attempts >= kubernetesCleanupMaxAttempts { + _ = r.deleteRedisCleanupJob(ctx, job) + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("the cleanup Job failed %d times: %s", attempts, jobFailureMessage(job))), nil + } + base := backup.DeepCopy() + if backup.Annotations == nil { + backup.Annotations = map[string]string{} + } + backup.Annotations[kubernetesCleanupAttemptsAnnotation] = strconv.Itoa(attempts) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + _ = r.deleteRedisCleanupJob(ctx, job) + return ctrl.Result{RequeueAfter: time.Duration(attempts) * 30 * time.Second}, nil + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } +} diff --git a/internal/backupcontroller/kubernetesstrategy_controller_test.go b/internal/backupcontroller/kubernetesstrategy_controller_test.go new file mode 100644 index 000000000..590f82ae9 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_controller_test.go @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "strconv" + "strings" + "testing" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +const ( + k8sTestNS = "tenant-a" + k8sTestApp = "c1" + k8sTestRelease = "kubernetes-c1" +) + +var kubernetesAppGVR = schema.GroupVersionResource{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Resource: "kubernetes"} + +func testCertPEM(t *testing.T, isCA bool, notAfter time.Time) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test"}, + NotBefore: notAfter.Add(-time.Hour), + NotAfter: notAfter, + IsCA: isCA, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func unstr(gvk schema.GroupVersionKind, ns, name string, fields map[string]any) *unstructured.Unstructured { + u := &unstructured.Unstructured{Object: map[string]any{}} + for k, v := range fields { + u.Object[k] = v + } + u.SetGroupVersionKind(gvk) + u.SetNamespace(ns) + u.SetName(name) + return u +} + +func TestKubernetesCertificateWarnings(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + secrets := []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "ca"}, Data: map[string][]byte{ + "ca.crt": testCertPEM(t, true, now.Add(10*24*time.Hour)), + "ca.key": []byte("not a certificate"), + }}, + {ObjectMeta: metav1.ObjectMeta{Name: "leaf-soon"}, Data: map[string][]byte{ + "tls.crt": testCertPEM(t, false, now.Add(24*time.Hour)), + }}, + {ObjectMeta: metav1.ObjectMeta{Name: "leaf-later"}, Data: map[string][]byte{ + "tls.crt": testCertPEM(t, false, now.Add(10*24*time.Hour)), + }}, + } + + got := kubernetesCertificateWarnings(secrets, now, 30*24*time.Hour, 2*24*time.Hour) + if len(got) != 2 { + t.Fatalf("got %d warnings %v, want 2", len(got), got) + } + if !strings.HasPrefix(got[0], "CA ca/ca.crt") || !strings.HasPrefix(got[1], "certificate leaf-soon/tls.crt") { + t.Fatalf("unexpected warnings %v", got) + } + + if got := kubernetesCertificateWarnings(secrets, now, time.Hour, time.Hour); len(got) != 0 { + t.Fatalf("want no warnings inside a short window, got %v", got) + } +} + +func TestKubernetesHelmReleaseNames(t *testing.T) { + mdGVK := schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"} + md := func(name, cluster, owner string) unstructured.Unstructured { + u := unstr(mdGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"clusterName": cluster}}) + u.SetAnnotations(map[string]string{helmReleaseNameAnnotation: owner}) + return *u + } + got := kubernetesHelmReleaseNames([]unstructured.Unstructured{ + md("a", k8sTestRelease, "kubernetes-nodes-c1-a"), + md("b", k8sTestRelease, "kubernetes-nodes-c1-b"), + md("legacy", k8sTestRelease, k8sTestRelease), + md("other", "kubernetes-c2", "kubernetes-nodes-c2-a"), + }, k8sTestRelease) + want := []string{k8sTestRelease, "kubernetes-nodes-c1-a", "kubernetes-nodes-c1-b"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestParseKubernetesRestoreOptions(t *testing.T) { + o, err := parseKubernetesRestoreOptions(nil) + if err != nil || !o.restoreEtcd() || !o.restoreVolumes() { + t.Fatalf("defaults: %+v %v", o, err) + } + o, err = parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"volumes":false}`)}) + if err != nil || !o.restoreEtcd() || o.restoreVolumes() { + t.Fatalf("etcd only: %+v %v", o, err) + } + if _, err := parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"etcd":false,"volumes":false}`)}); err == nil { + t.Fatal("want an error when both halves are disabled") + } +} + +func TestKubernetesVeleroRestoreSpecPhases(t *testing.T) { + strategy := &strategyv1alpha1.Kubernetes{} + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS}, + Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: k8sTestApp}}, + } + secrets, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseSecrets, k8sTestRelease) + if err != nil { + t.Fatal(err) + } + if strings.Join(secrets.IncludedResources, ",") != "secrets" || len(secrets.OrLabelSelectors) != 2 { + t.Fatalf("secrets phase: %+v", secrets) + } + volumes, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseVolumes, k8sTestRelease) + if err != nil { + t.Fatal(err) + } + if volumes.OrLabelSelectors[0].MatchLabels[clusterNameLabel] != k8sTestRelease || volumes.RestorePVs == nil || !*volumes.RestorePVs { + t.Fatalf("volumes phase: %+v", volumes) + } + if _, err := kubernetesVeleroRestoreSpec(strategy, backup, "bogus", k8sTestRelease); err == nil { + t.Fatal("want an error for an unknown phase") + } +} + +type kubernetesTestEnv struct { + backup *BackupJobReconciler + restore *RestoreJobReconciler + client client.Client +} + +// newKubernetesTestEnv seeds a live tenant cluster: the application, its +// HelmReleases, TenantControlPlane, DataStore, KamajiControlPlane, one worker +// pool and one data DataVolume. +const k8sTestControlPlaneUID = "6b0c4f1e-2d3a-4e5f-8a9b-0c1d2e3f4a5b" + +func newKubernetesTestEnv(t *testing.T, objs ...client.Object) *kubernetesTestEnv { + t.Helper() + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = backupsv1alpha1.AddToScheme(s) + _ = strategyv1alpha1.AddToScheme(s) + _ = velerov1.AddToScheme(s) + + listKinds := map[schema.GroupVersionResource]string{ + kubernetesAppGVR: "KubernetesList", + helmReleaseGVR: "HelmReleaseList", + tenantControlPlaneGVR: "TenantControlPlaneList", + kamajiDataStoreGVR: "DataStoreList", + kamajiControlPlaneGVR: "KamajiControlPlaneList", + machineDeploymentGVR: "MachineDeploymentList", + machineGVR: "MachineList", + dataVolumeGVR: "DataVolumeList", + deploymentGVR: "DeploymentList", + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(s, listKinds) + seed := func(gvr schema.GroupVersionResource, u *unstructured.Unstructured) { + if err := dyn.Tracker().Create(gvr, u, u.GetNamespace()); err != nil { + t.Fatalf("seed %s: %v", u.GetName(), err) + } + } + appGVK := schema.GroupVersionKind{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Kind: kubernetesAppKind} + seed(kubernetesAppGVR, unstr(appGVK, k8sTestNS, k8sTestApp, map[string]any{"spec": map[string]any{}})) + hrGVK := schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"} + for _, name := range []string{k8sTestRelease, "kubernetes-nodes-c1-md0"} { + hr := unstr(hrGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"interval": "5m"}}) + hr.SetLabels(map[string]string{"apps.cozystack.io/application.name": name, "velero.io/backup-name": "old"}) + seed(helmReleaseGVR, hr) + } + tcp := unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{ + "status": map[string]any{"storage": map[string]any{ + "driver": "etcd", + "dataStoreName": "tenant-a", + "setup": map[string]any{"schema": "tenant_a_kubernetes_c1"}, + "certificate": map[string]any{"secretName": k8sTestRelease + "-datastore-certificate"}, + }}, + }) + tcp.SetUID(k8sTestControlPlaneUID) + seed(tenantControlPlaneGVR, tcp) + seed(kamajiDataStoreGVR, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "DataStore"}, "", "tenant-a", map[string]any{ + "spec": map[string]any{"endpoints": []any{"etcd.tenant-a.svc:2379"}}, + })) + seed(kamajiControlPlaneGVR, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + })) + md := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"}, k8sTestNS, k8sTestRelease+"-md0", map[string]any{ + "spec": map[string]any{"clusterName": k8sTestRelease, "replicas": int64(3)}, + }) + md.SetAnnotations(map[string]string{helmReleaseNameAnnotation: "kubernetes-nodes-c1-md0"}) + seed(machineDeploymentGVR, md) + dv := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-1234", nil) + dv.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + seed(dataVolumeGVR, dv) + seed(deploymentGVR, unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease+"-cluster-autoscaler", map[string]any{ + "spec": map[string]any{"replicas": int64(1)}, + })) + + base := []client.Object{ + &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "tenant-a.bj1-xyz"}, + Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseCompleted}, + }, + &strategyv1alpha1.Kubernetes{ + ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-kubernetes"}, + Spec: strategyv1alpha1.KubernetesSpec{ + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "etcd", + Image: "img", + Args: []string{"{{ .Mode }} {{ .ObjectKey }} {{ .Datastore.Prefix }} {{ .Datastore.Endpoints }} {{ .Datastore.CertificateSecret }} {{ .Datastore.SourcePrefix }}"}, + }}}}, + Velero: strategyv1alpha1.VeleroTemplate{Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"{{ .Application.metadata.namespace }}"}, + TTL: metav1.Duration{Duration: 720 * time.Hour}, + }}, + }, + }, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-ca"}, Data: map[string][]byte{ + "ca.crt": testCertPEM(t, true, time.Now().Add(24*time.Hour)), + }}, + } + c := clientfake.NewClientBuilder().WithScheme(s). + WithObjects(append(base, objs...)...). + WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). + Build() + mapper := &mockRESTMapper{mapping: &meta.RESTMapping{Resource: kubernetesAppGVR, GroupVersionKind: appGVK, Scope: meta.RESTScopeNamespace}} + return &kubernetesTestEnv{ + backup: &BackupJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)}, + restore: &RestoreJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)}, + client: c, + } +} + +func k8sResolved() *ResolvedBackupConfig { + return &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new("strategy.backups.cozystack.io"), Kind: strategyv1alpha1.KubernetesStrategyKind, Name: "cozy-default-kubernetes"}, + Parameters: map[string]string{"p": "v"}, + } +} + +func TestReconcileKubernetesBackup(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + get := func() *backupsv1alpha1.BackupJob { + out := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil { + t.Fatal(err) + } + return out + } + step := func() { + t.Helper() + if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil { + t.Fatal(err) + } + } + + step() // StartedAt + step() // certificates + cond := meta.FindStatusCondition(get().Status.Conditions, kubernetesCondCertificates) + if cond == nil || cond.Status != metav1.ConditionTrue || !strings.Contains(cond.Message, k8sTestRelease+"-ca/ca.crt") { + t.Fatalf("certificate condition: %+v", cond) + } + + step() // dump Job + job := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1-backup"}, job); err != nil { + t.Fatalf("dump Job: %v", err) + } + args := job.Spec.Template.Spec.Containers[0].Args[0] + want := "backup tenant-a/c1/bj1.etcd.json tenant_a_kubernetes_c1 https://etcd.tenant-a.svc:2379 kubernetes-c1-datastore-certificate tenant_a_kubernetes_c1" + if args != want { + t.Fatalf("rendered args %q, want %q", args, want) + } + step() + vbs := &velerov1.BackupList{} + _ = env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}) + if len(vbs.Items) != 0 { + t.Fatal("Velero Backup created before the dump Job completed") + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := env.client.Status().Update(ctx, job); err != nil { + t.Fatal(err) + } + // A claim the CSI driver created without a DataVolume is still one an + // in-place restore has to clear before Velero writes it back. + if err := env.client.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: k8sTestNS, Name: "pvc-plain", Labels: map[string]string{clusterNameLabel: k8sTestRelease}, + }}); err != nil { + t.Fatal(err) + } + step() + if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}); err != nil || len(vbs.Items) != 1 { + t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err) + } + // A volume created while Velero runs may be missing from its backup, so a + // restore of this backup must never delete it. + late := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-late", nil) + late.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + if _, err := env.backup.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Create(ctx, late, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if ns := vbs.Items[0].Spec.IncludedNamespaces; len(ns) != 1 || ns[0] != k8sTestNS { + t.Fatalf("Velero spec not rendered: %v", ns) + } + talosCA := &corev1.Secret{} + _ = env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}, talosCA) + if talosCA.Labels[helmInstanceLabel] != k8sTestRelease { + t.Fatalf("talos CA not labelled for the Velero selector: %v", talosCA.Labels) + } + + vb := vbs.Items[0] + vb.Status.Phase = velerov1.BackupPhaseCompleted + if err := env.client.Update(ctx, &vb); err != nil { + t.Fatal(err) + } + // A pool that is suspended while the backup runs must not come back + // suspended when the cluster is rebuilt from it. + if _, err := env.backup.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(ctx, "kubernetes-nodes-c1-md0", types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + step() + if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded || got.Status.BackupRef == nil { + t.Fatalf("BackupJob not succeeded: %+v", got.Status) + } + backup := &backupsv1alpha1.Backup{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1"}, backup); err != nil { + t.Fatal(err) + } + md := backup.Spec.DriverMetadata + if md[veleroBackupNameMetadataKey] != vb.Name || md[kubernetesDataVolumesMetaKey] != "pvc-1234,pvc-plain" || + md[kubernetesDatastorePrefixMetaKey] != "tenant_a_kubernetes_c1" || md[kubernetesParamPrefix+"p"] != "v" || + md[kubernetesCertWarningsMetaKey] == "" || md[kubernetesControlPlaneUIDMetaKey] != k8sTestControlPlaneUID { + t.Fatalf("driver metadata: %v", md) + } + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil || len(releases) != 2 { + t.Fatalf("recorded HelmReleases: %v", releases) + } + raw, _ := json.Marshal(releases) + if strings.Contains(string(raw), "velero.io/") || strings.Contains(string(raw), "suspend") { + t.Fatalf("Velero labels or the suspend flag leaked into the recorded HelmReleases: %s", raw) + } +} + +func k8sBackupForRestore() *backupsv1alpha1.Backup { + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp}, + StrategyRef: k8sResolved().StrategyRef, + DriverMetadata: map[string]string{ + veleroBackupNameMetadataKey: "tenant-a.bj1-xyz", + kubernetesObjectMetaKey: "tenant-a/c1/bj1.etcd.json", + kubernetesDatastorePrefixMetaKey: "tenant_a_kubernetes_c1", + kubernetesControlPlaneUIDMetaKey: k8sTestControlPlaneUID, + kubernetesDatastoreEndpointsMetaKey: "https://etcd.tenant-a.svc:2379", + kubernetesDataVolumesMetaKey: "pvc-1234", + kubernetesHelmReleasesMetaKey: `[{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-c1"},"spec":{"interval":"5m"}}]`, + }, + }, + } +} + +// k8sRestoreRun drives one RestoreJob through the test environment. +type k8sRestoreRun struct { + t *testing.T + env *kubernetesTestEnv + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup +} + +func newK8sRestoreRun(t *testing.T, options string, backup *backupsv1alpha1.Backup) *k8sRestoreRun { + t.Helper() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + } + if options != "" { + rj.Spec.Options = &runtime.RawExtension{Raw: []byte(options)} + } + return &k8sRestoreRun{t: t, env: newKubernetesTestEnv(t, backup, rj), rj: rj, backup: backup} +} + +func (r *k8sRestoreRun) get() *backupsv1alpha1.RestoreJob { + r.t.Helper() + out := &backupsv1alpha1.RestoreJob{} + if err := r.env.client.Get(context.Background(), client.ObjectKeyFromObject(r.rj), out); err != nil { + r.t.Fatal(err) + } + return out +} + +func (r *k8sRestoreRun) step() { + r.t.Helper() + if _, err := r.env.restore.reconcileKubernetesRestore(context.Background(), r.get(), r.backup); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) finishJob(name string, condType batchv1.JobConditionType) { + r.t.Helper() + job := &batchv1.Job{} + if err := r.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: name}, job); err != nil { + r.t.Fatalf("Job %s: %v", name, err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: condType, Status: corev1.ConditionTrue, Message: "boom"}} + if err := r.env.client.Status().Update(context.Background(), job); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) finishVeleroRestore(phase velerov1.RestorePhase) { + r.t.Helper() + list := &velerov1.RestoreList{} + if err := r.env.client.List(context.Background(), list); err != nil || len(list.Items) == 0 { + r.t.Fatalf("Velero Restores: %v %v", len(list.Items), err) + } + vr := list.Items[len(list.Items)-1] + vr.Status.Phase = phase + if err := r.env.client.Update(context.Background(), &vr); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) obj(gvr schema.GroupVersionResource, name string) *unstructured.Unstructured { + r.t.Helper() + u, err := r.env.restore.Interface.Resource(gvr).Namespace(k8sTestNS).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + r.t.Fatalf("get %s %s: %v", gvr.Resource, name, err) + } + return u +} + +func (r *k8sRestoreRun) replicas(gvr schema.GroupVersionResource, name string) (int64, bool) { + n, found, _ := unstructured.NestedInt64(r.obj(gvr, name).Object, "spec", "replicas") + return n, found +} + +func (r *k8sRestoreRun) suspended(name string) bool { + s, _, _ := unstructured.NestedBool(r.obj(helmReleaseGVR, name).Object, "spec", "suspend") + return s +} + +func (r *k8sRestoreRun) addMachine(name string) { + r.t.Helper() + m := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "Machine"}, k8sTestNS, name, nil) + m.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + if _, err := r.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Create(context.Background(), m, metav1.CreateOptions{}); err != nil { + r.t.Fatal(err) + } +} + +// verifyAndCapture runs the preflight and the mode decision. +func (r *k8sRestoreRun) verifyAndCapture(etcd bool) { + r.t.Helper() + r.step() // StartedAt + r.step() // preflight + if etcd { + r.finishJob("rj-verify", batchv1.JobComplete) + r.step() + } + if !meta.IsStatusConditionTrue(r.get().Status.Conditions, kubernetesCondVerified) { + r.t.Fatalf("backup not verified: %+v", r.get().Status) + } + r.step() // capture +} + +func TestReconcileKubernetesRestoreInPlaceDefault(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + + run.step() // stop: a worker is still there + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatal("control plane scaled down while a worker Machine still needs it to delete its Node") + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 0 { + t.Fatalf("worker pool not scaled down: %d", n) + } + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 { + t.Fatalf("autoscaler not scaled down: %d", n) + } + if _, ok := run.obj(machineGVR, "w-1").GetAnnotations()[machineExcludeDrainAnnotation]; !ok { + t.Fatal("Machine not released from draining") + } + if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondStopped); c == nil || c.Status != metav1.ConditionFalse || !strings.Contains(c.Message, "Machine") { + t.Fatalf("stop progress not reported: %+v", c) + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatal("data DataVolume deleted while a worker may still mount it") + } + + if err := run.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Delete(context.Background(), "w-1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + run.step() // stop settles + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err == nil { + t.Fatal("backed-up data DataVolume not deleted once the workers were gone") + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane not scaled down after the workers were gone") + } + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster not reported stopped") + } + + run.step() // etcd Job + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() // etcd done + run.step() // volumes Velero Restore + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() // volumes done + run.step() // resume + + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", got.Status) + } + for _, c := range []struct { + gvr schema.GroupVersionResource + name string + want int64 + }{ + {kamajiControlPlaneGVR, k8sTestRelease, 2}, + {machineDeploymentGVR, k8sTestRelease + "-md0", 3}, + {deploymentGVR, k8sTestRelease + "-cluster-autoscaler", 1}, + } { + if n, _ := run.replicas(c.gvr, c.name); n != c.want { + t.Errorf("%s %s replicas = %d, want %d", c.gvr.Resource, c.name, n, c.want) + } + if _, left := run.obj(c.gvr, c.name).GetAnnotations()[kubernetesReplicasBeforeAnnotation]; left { + t.Errorf("%s %s keeps the restore's replica record", c.gvr.Resource, c.name) + } + } + if run.suspended(k8sTestRelease) || run.suspended("kubernetes-nodes-c1-md0") { + t.Fatal("HelmRelease left suspended") + } +} + +func TestReconcileKubernetesRestoreInPlaceEtcdOnly(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatalf("control plane not scaled down: %d", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("workers touched by an etcd-only restore: %d", n) + } + run.step() + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane replicas not restored: %d", n) + } + list := &velerov1.RestoreList{} + _ = run.env.client.List(context.Background(), list) + if len(list.Items) != 0 { + t.Fatal("an etcd-only restore created a Velero Restore") + } +} + +func TestReconcileKubernetesRestoreKeepsPreSuspendedRelease(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + patch := []byte(`{"spec":{"suspend":true}}`) + if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(context.Background(), "kubernetes-nodes-c1-md0", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + run.step() + run.step() + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if run.get().Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", run.get().Status) + } + if !run.suspended("kubernetes-nodes-c1-md0") { + t.Fatal("a HelmRelease suspended before the restore was resumed by it") + } + if run.suspended(k8sTestRelease) { + t.Fatal("the cluster HelmRelease the restore suspended was left suspended") + } +} + +func TestReconcileKubernetesRestoreEtcdFailureLeavesControlPlaneStopped(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() + run.step() + run.addJobPod("rj-restore", kubernetesJobWriting) + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) { + t.Fatalf("want a failed restore with the etcd write marked failed: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane brought back up on a possibly partial etcd prefix") + } + if !strings.Contains(got.Status.Message+fmtConditions(got), ".pre-restore") { + t.Fatalf("failure does not point at the pre-restore copy: %+v", got.Status) + } + + run.env.restore.cleanupKubernetesRestore(context.Background(), got) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("deleting the RestoreJob brought the control plane up on a possibly partial prefix") + } +} + +func fmtConditions(rj *backupsv1alpha1.RestoreJob) string { + var b strings.Builder + for _, c := range rj.Status.Conditions { + b.WriteString(c.Message) + } + return b.String() +} + +func TestCleanupKubernetesRestoreResumesStoppedCluster(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop: workers gone, volumes deleted, control plane stopped + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatalf("control plane not stopped: %d", n) + } + run.env.restore.cleanupKubernetesRestore(context.Background(), run.get()) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("deleting the RestoreJob left the control plane at %d replicas", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("deleting the RestoreJob left the worker pool at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("deleting the RestoreJob left the HelmRelease suspended") + } +} + +func TestReconcileKubernetesRestoreStopDeadline(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + run.step() // stop starts, a worker remains + rj := run.get() + cond := meta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped) + cond.LastTransitionTime = metav1.NewTime(time.Now().Add(-restorePreparationDeadline - time.Minute)) + if err := run.env.client.Status().Update(context.Background(), rj); err != nil { + t.Fatal(err) + } + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a stop that never settles must fail the restore: %+v", got.Status) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool not resumed after the stop timed out: %d", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("HelmRelease left suspended after the stop timed out") + } +} + +func TestReconcileKubernetesRestoreVerifiesBeforeChangingAnything(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "expired" + run := newK8sRestoreRun(t, "", backup) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a missing Velero backup must fail the restore: %+v", got.Status) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster touched before the backup was verified") + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatal("data DataVolume deleted before the backup was verified") + } + + backup = k8sBackupForRestore() + backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "partial" + run = newK8sRestoreRun(t, "", backup) + if err := run.env.client.Create(context.Background(), &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "partial"}, + Status: velerov1.BackupStatus{Phase: velerov1.BackupPhasePartiallyFailed}, + }); err != nil { + t.Fatal(err) + } + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a Velero backup that did not complete must fail the restore: %+v", got.Status) + } + + run = newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.step() + run.step() + run.finishJob("rj-verify", batchv1.JobFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a missing etcd dump must fail the restore: %+v", got.Status) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster touched before the dump was verified") + } +} + +func k8sBackupOfDeletedCluster() *backupsv1alpha1.Backup { + backup := k8sBackupForRestore() + backup.Spec.ApplicationRef.Name = "gone" + backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = `[` + + `{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-gone"},"spec":{"interval":"5m"}},` + + `{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-nodes-gone-md0"},"spec":{"interval":"5m"}}]` + return backup +} + +// TestReconcileKubernetesRestoreRecreate rebuilds a deleted cluster whose new +// TenantControlPlane gets a fresh, UID-keyed schema: the dump must be written +// under that schema, and the worker pools come back only after it is. +func TestReconcileKubernetesRestoreRecreate(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeRecreate || state.Release != "kubernetes-gone" { + t.Fatalf("want recreate mode, got %+v", state) + } + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // volumes + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + + run.step() // cluster HelmRelease, waiting for the TenantControlPlane + dyn := run.env.restore.Interface + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-gone", metav1.GetOptions{}); err != nil { + t.Fatalf("cluster HelmRelease not recreated: %v", err) + } + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err == nil { + t.Fatal("worker pool recreated before the etcd state was written back") + } + if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondRecreated); c == nil || c.Status != metav1.ConditionFalse { + t.Fatalf("wait for the new control plane not reported: %+v", c) + } + run.step() // a later poll, the control plane still coming up + if ann := run.obj(helmReleaseGVR, "kubernetes-gone").GetAnnotations(); ann["reconcile.fluxcd.io/forceAt"] != "" { + t.Fatalf("the cluster HelmRelease the rebuild created is forced through a Helm upgrade on a poll: %v", ann) + } + + const newSchema = "0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10" + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": newSchema}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stop it for the etcd write + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 0 { + t.Fatalf("new control plane not stopped before the etcd write: %d", n) + } + run.step() // etcd Job + job := &batchv1.Job{} + if err := run.env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err != nil { + t.Fatal(err) + } + args := job.Spec.Template.Spec.Containers[0].Args[0] + if !strings.Contains(args, " "+newSchema+" ") || !strings.HasSuffix(args, " tenant_a_kubernetes_c1") || !strings.Contains(args, "kubernetes-gone-datastore-certificate") { + t.Fatalf("restore Job does not write the dump under the new cluster's schema: %q", args) + } + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() // resume, worker pools + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("rebuild not succeeded: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 { + t.Fatalf("new control plane not started again: %d", n) + } + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err != nil { + t.Fatalf("worker pool not recreated: %v", err) + } +} + +func TestReconcileKubernetesRestoreRecreateSecretsFailure(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + run.step() // secrets restore + list := &velerov1.RestoreList{} + if err := run.env.client.List(context.Background(), list); err != nil || len(list.Items) != 1 { + t.Fatalf("Velero Restores: %d %v", len(list.Items), err) + } + vr := list.Items[0] + if vr.Labels[kubernetesRestorePhaseLabel] != kubernetesPhaseSecrets || vr.Spec.BackupName != "tenant-a.bj1-xyz" || vr.Spec.ResourceModifier == nil { + t.Fatalf("secrets Restore: %+v", vr) + } + cm := &corev1.ConfigMap{} + if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: veleroNamespace, Name: vr.Spec.ResourceModifier.Name}, cm); err != nil { + t.Fatal(err) + } + if !strings.Contains(cm.Data["resource-modifier-rules.yaml"], "ownerReferences: null") { + t.Fatalf("modifiers do not strip ownerReferences: %s", cm.Data["resource-modifier-rules.yaml"]) + } + run.finishVeleroRestore(velerov1.RestorePhaseFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a failed Velero Restore must fail the RestoreJob: %+v", got.Status) + } +} + +func TestReconcileKubernetesRestoreRecreateRejectsBadReleaseRecord(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.ApplicationRef.Name = "gone" + backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = "not json" + run := newK8sRestoreRun(t, "", backup) + run.verifyAndCapture(true) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("an unreadable HelmRelease record must fail the rebuild: %+v", got.Status) + } +} + +func TestReconcileKubernetesRestoreRejectsRename(t *testing.T) { + backup := k8sBackupForRestore() + run := newK8sRestoreRun(t, "", backup) + rj := run.get() + rj.Spec.TargetApplicationRef = &corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: "other"} + if _, err := run.env.restore.reconcileKubernetesRestore(context.Background(), rj, backup); err != nil { + t.Fatal(err) + } + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("restore into another application must fail: %+v", got.Status) + } +} + +func TestResolveKubernetesDatastoreRejectsNonEtcd(t *testing.T) { + env := newKubernetesTestEnv(t) + dyn := env.backup.Interface + tcp, _ := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(context.Background(), k8sTestRelease, metav1.GetOptions{}) + _ = unstructured.SetNestedField(tcp.Object, "PostgreSQL", "status", "storage", "driver") + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(context.Background(), tcp, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := resolveKubernetesDatastore(context.Background(), dyn, k8sTestNS, k8sTestApp); err == nil { + t.Fatal("want an error for a non-etcd datastore") + } +} + +func TestCleanupKubernetesBackupGivesUpAfterRepeatedFailures(t *testing.T) { + ctx := context.Background() + backup := k8sBackupForRestore() + backup.Annotations = map[string]string{kubernetesCleanupAttemptsAnnotation: strconv.Itoa(kubernetesCleanupMaxAttempts - 1)} + failed := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1-cleanup"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, + } + env := newKubernetesTestEnv(t, backup, failed) + br := &BackupReconciler{Client: env.client, Scheme: env.backup.Scheme, Recorder: record.NewFakeRecorder(10)} + res, err := br.cleanupKubernetesBackup(ctx, backup) + if err != nil { + t.Fatal(err) + } + if res.RequeueAfter != 0 { + t.Fatalf("cleanup keeps retrying after %d failures: %+v", kubernetesCleanupMaxAttempts, res) + } +} + +func (r *k8sRestoreRun) addJobPod(job, message string) { + r.t.Helper() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: job + "-pod", Labels: map[string]string{"job-name": job}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Message: message + "\n"}}, + }}}, + } + if err := r.env.client.Create(context.Background(), pod); err != nil { + r.t.Fatal(err) + } +} + +func TestReconcileKubernetesRestoreFailureBeforeWriteResumes(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + run.addJobPod("rj-restore", "") + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) { + t.Fatalf("a Job that never touched the prefix must fail without marking it partial: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left stopped although the prefix was never changed: %d", n) + } +} + +func TestCleanupKubernetesRestoreKeepsClusterStoppedWhileRestoreJobRuns(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job, still running + run.env.restore.cleanupKubernetesRestore(context.Background(), run.get()) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("deleting the RestoreJob brought the control plane up while the restore Job may be rewriting the prefix") + } + job := &batchv1.Job{} + if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err == nil { + t.Fatal("the running restore Job was left to keep writing") + } +} + +func TestReconcileKubernetesRestoreIgnoresModeAnnotation(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + rj := run.get() + rj.Annotations = map[string]string{"kubernetes.strategy.backups.cozystack.io/restore-state": `{"mode":"recreate","release":"kubernetes-c1"}`} + if err := run.env.client.Update(context.Background(), rj); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeInPlace { + t.Fatalf("a live cluster must be restored in place whatever the RestoreJob claims: %+v", state) + } +} + +func TestReconcileKubernetesBackupDiscardsDumpWhenVeleroFails(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj2"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + step := func() { + t.Helper() + got := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), got); err != nil { + t.Fatal(err) + } + if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil { + t.Fatal(err) + } + } + step() + step() + step() + dump := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-backup"}, dump); err != nil { + t.Fatal(err) + } + dump.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := env.client.Status().Update(ctx, dump); err != nil { + t.Fatal(err) + } + step() + vbs := &velerov1.BackupList{} + if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj2"}); err != nil || len(vbs.Items) != 1 { + t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err) + } + vb := vbs.Items[0] + vb.Status.Phase = velerov1.BackupPhaseFailed + if err := env.client.Update(ctx, &vb); err != nil { + t.Fatal(err) + } + step() + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob not failed: %+v", got.Status) + } + discard := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-discard"}, discard); err != nil { + t.Fatalf("the orphaned etcd dump is not scheduled for deletion: %v", err) + } + if args := discard.Spec.Template.Spec.Containers[0].Args[0]; !strings.HasPrefix(args, "cleanup tenant-a/c1/bj2.etcd.json") { + t.Fatalf("discard Job does not delete this run's dump: %q", args) + } +} + +func TestReconcileKubernetesRestoreRecreateFailureBeforeWriteStartsControlPlane(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // cluster HelmRelease + dyn := run.env.restore.Interface + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": "uid-1"}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stopped + run.step() // etcd Job + run.addJobPod("rj-restore", "") + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("rebuild not failed: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 { + t.Fatalf("the rebuilt control plane is left stopped although its prefix was never changed: %d", n) + } +} + +func TestReconcileKubernetesRestoreJobThatNeverRanResumes(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}, + }}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.step() // still waiting on the Job + if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) { + t.Fatal("a container that never ran was taken for a started Job") + } + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left stopped after a restore Job that never ran: %d", n) + } +} + +func TestReconcileKubernetesRestoreJobPodsGoneAfterStart(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) { + t.Fatal("a running restore Job was not recorded as started") + } + // The Job deadline deletes its Pods and fails it. + if err := run.env.client.Delete(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane brought up after a restore Job that ran and left no trace of how far it got") + } +} + +func TestReconcileKubernetesRestoreRetriesStopAfterAPIError(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + failed := false + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("patch", "kamajicontrolplanes", func(clienttesting.Action) (bool, runtime.Object, error) { + if failed { + return false, nil, nil + } + failed = true + return true, nil, fmt.Errorf("etcdserver: request timed out") + }) + run.step() + got := run.get() + if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("one API error aborted the restore: %+v", got.Status) + } + if c := meta.FindStatusCondition(got.Status.Conditions, kubernetesCondStopped); c == nil || !strings.Contains(c.Message, "API error") { + t.Fatalf("the retried API error is not reported: %+v", c) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("the stop did not go through once the API answered again") + } +} + +// driveRecreateToEtcdJob runs a rebuild up to the etcd restore Job, the new +// TenantControlPlane reporting schema. +func driveRecreateToEtcdJob(t *testing.T, run *k8sRestoreRun, schemaName string) { + t.Helper() + ctx := context.Background() + run.verifyAndCapture(true) + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // volumes + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // cluster HelmRelease + dyn := run.env.restore.Interface + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": schemaName}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stopped + run.step() // etcd Job +} + +func TestReconcileKubernetesRestoreRecreateForcesSurvivingPool(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + // Deleting only the Kubernetes application leaves its KubernetesNodes + // pool's HelmRelease behind, while the pool's MachineDeployment went + // with the CAPI Cluster. + pool := unstr(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, k8sTestNS, "kubernetes-nodes-gone-md0", map[string]any{"spec": map[string]any{"interval": "5m"}}) + if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Create(ctx, pool, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + driveRecreateToEtcdJob(t, run, "uid-1") + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("rebuild not succeeded: %+v", got.Status) + } + ann := run.obj(helmReleaseGVR, "kubernetes-nodes-gone-md0").GetAnnotations() + if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/forceAt"] != ann["reconcile.fluxcd.io/requestedAt"] { + t.Fatalf("the surviving pool is not forced to recreate its MachineDeployment: %v", ann) + } +} + +func TestReconcileKubernetesBackupRefusesClusterStoppedByRestore(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj3"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesReplicasBeforeAnnotation + `":"2"}},"spec":{"replicas":0}}`) + if _, err := env.backup.Interface.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Patch(ctx, k8sTestRelease, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + for range 2 { + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil { + t.Fatal(err) + } + } + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a cluster stopped by an unfinished restore was backed up: %+v", got.Status) + } + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj3-backup"}, &batchv1.Job{}); err == nil { + t.Fatal("the etcd prefix of a stopped cluster was dumped") + } +} + +func TestStopKubernetesClusterWaitsForTerminatingControlPlane(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + cp := unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease, map[string]any{ + "spec": map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"kamaji.clastix.io/name": k8sTestRelease}}}, + "status": map[string]any{"replicas": int64(0)}, + }) + if _, err := run.env.restore.Interface.Resource(deploymentGVR).Namespace(k8sTestNS).Create(ctx, cp, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "apiserver-0", Labels: map[string]string{"kamaji.clastix.io/name": k8sTestRelease}}} + if err := run.env.client.Create(ctx, pod); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + run.step() + if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster reported stopped while a control-plane Pod is still terminating") + } + if err := run.env.client.Delete(ctx, pod); err != nil { + t.Fatal(err) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster not reported stopped once its control plane was gone") + } +} + +func TestKubernetesJobReachedWriteConservativeBranches(t *testing.T) { + for _, tc := range []struct { + name string + status corev1.ContainerStatus + want bool + }{ + {"running", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, true}, + {"restarted after writing", corev1.ContainerStatus{Name: "etcd", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}}, + LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Message: kubernetesJobWriting}}}, true}, + {"never ran", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, false}, + {"failed before writing", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1}}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "p", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{tc.status}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + got, err := run.env.restore.kubernetesJobReachedWrite(context.Background(), run.get(), "rj-restore") + if err != nil || got != tc.want { + t.Fatalf("reached write = %v (%v), want %v", got, err, tc.want) + } + }) + } +} + +// TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed covers a Cluster +// API that no longer serves the version the driver lists. Reading that as a +// cluster without workers would let the volumes path delete DataVolumes that +// running workers still mount. +func TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed(t *testing.T) { + for _, tc := range []struct{ resource, kind string }{ + {"machinedeployments", "MachineDeployment"}, + {"machines", "Machine"}, + } { + t.Run(tc.resource, func(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.verifyAndCapture(true) + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", tc.resource, func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: tc.kind}, SearchedVersions: []string{"v1beta1"}} + }) + run.step() + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "Cluster API is not served") { + t.Fatalf("restore not failed on an unserved Cluster API: %+v", got.Status) + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatalf("backed-up DataVolume deleted: %v", err) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left at %d replicas", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool left at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease left suspended") + } + }) + } +} + +// TestReconcileKubernetesRestoreRefusesRecreatedCluster covers an application +// deleted after the backup and created again under the same name. Its new CA +// and service-account key come back only through a rebuild, so an in-place +// restore must refuse before it changes anything. Backups without a recorded +// UID are told apart by the datastore schema. +func TestReconcileKubernetesRestoreRefusesRecreatedCluster(t *testing.T) { + for _, tc := range []struct { + name string + edit func(map[string]string) + }{ + {"recorded UID", func(md map[string]string) { md[kubernetesControlPlaneUIDMetaKey] = "uid-of-the-deleted-cluster" }}, + {"schema of a backup without the UID", func(md map[string]string) { + delete(md, kubernetesControlPlaneUIDMetaKey) + md[kubernetesDatastorePrefixMetaKey] = "uid-of-the-deleted-cluster" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + backup := k8sBackupForRestore() + tc.edit(backup.Spec.DriverMetadata) + run := newK8sRestoreRun(t, "", backup) + run.verifyAndCapture(true) + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "deleted and created again") { + t.Fatalf("restore into a re-created cluster not refused: %+v", got.Status) + } + if kubernetesStateFromRestoreJob(got) != nil { + t.Fatal("restore mode recorded for a refused restore") + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease suspended by a refused restore") + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool scaled to %d by a refused restore", n) + } + }) + } +} + +// TestFailedKubernetesRestoreResumesWithoutClusterAPI covers Cluster API going +// unserved after the workers were stopped: the pools cannot be read back, but +// the autoscaler and the suspended HelmReleases must still be put back. +func TestFailedKubernetesRestoreResumesWithoutClusterAPI(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + run.step() // workers stopping, the Machine still there + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 { + t.Fatalf("autoscaler not stopped: %d", n) + } + for _, res := range []string{"machinedeployments", "machines"} { + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", res, func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: "MachineDeployment"}, SearchedVersions: []string{"v1beta1"}} + }) + } + run.step() + + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("restore not failed: %+v", got.Status) + } + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 1 { + t.Fatalf("autoscaler left at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease left suspended") + } +} + +// TestRenderKubernetesTemplateRefusesUnexpectedSchema pins the allowlist every +// strategy Job is rendered through: the schema becomes the prefix the restore +// deletes in the etcd the whole tenant shares. +func TestRenderKubernetesTemplateRefusesUnexpectedSchema(t *testing.T) { + tmpl := corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "etcd", Args: []string{"{{ .Datastore.Prefix }} {{ .Datastore.SourcePrefix }}"}}}}} + render := func(prefix, source string) error { + _, err := renderKubernetesTemplate(tmpl, k8sTestApp, k8sTestNS, kubernetesModeRestore, "key", kubernetesDatastore{Prefix: prefix, SourcePrefix: source}, nil) + return err + } + for _, ok := range []string{"0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10", "tenant_root_kubernetes_bk1"} { + if err := render(ok, ok); err != nil { + t.Errorf("schema %q refused: %v", ok, err) + } + } + if err := render("tenant_a_kubernetes_c1", ""); err != nil { + t.Errorf("a Job without a source schema refused: %v", err) + } + for _, bad := range []string{"", "a/b", "/", "..", "a..b", "a.b", "Tenant_a", "a b", "a\nb", "a*"} { + if render(bad, "tenant_a_kubernetes_c1") == nil { + t.Errorf("schema %q accepted", bad) + } + if bad != "" && render("tenant_a_kubernetes_c1", bad) == nil { + t.Errorf("source schema %q accepted", bad) + } + } +} + +// TestReconcileKubernetesRestoreRefusesEditedPrefix covers a Backup whose +// recorded prefix was edited: the restore fails in its preflight, before any +// Job runs against the shared etcd or the cluster is stopped. +func TestReconcileKubernetesRestoreRefusesEditedPrefix(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] = "tenant_a_kubernetes_c1/.." + run := newK8sRestoreRun(t, "", backup) + run.step() // StartedAt + run.step() // preflight + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") { + t.Fatalf("restore from an edited prefix not refused: %+v", got.Status) + } + jobs := &batchv1.JobList{} + if err := run.env.client.List(context.Background(), jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("Jobs created for an edited prefix: %d %v", len(jobs.Items), err) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease suspended") + } +} + +// TestReconcileKubernetesBackupRefusesUnexpectedSchema covers a live +// TenantControlPlane reporting a schema outside the allowlist. +func TestReconcileKubernetesBackupRefusesUnexpectedSchema(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + tcp, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(ctx, k8sTestRelease, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(tcp.Object, "tenant_a/kubernetes_c1", "status", "storage", "setup", "schema"); err != nil { + t.Fatal(err) + } + if _, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(ctx, tcp, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + get := func() *backupsv1alpha1.BackupJob { + out := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil { + t.Fatal(err) + } + return out + } + for i := 0; i < 4 && get().Status.Phase != backupsv1alpha1.BackupJobPhaseFailed; i++ { + if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil { + t.Fatal(err) + } + } + if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") { + t.Fatalf("backup of an unexpected schema not refused: %+v", got.Status) + } + jobs := &batchv1.JobList{} + if err := env.client.List(ctx, jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("dump Job created for an unexpected schema: %d %v", len(jobs.Items), err) + } +} diff --git a/internal/backupcontroller/kubernetesstrategy_restore.go b/internal/backupcontroller/kubernetesstrategy_restore.go new file mode 100644 index 000000000..830a25927 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_restore.go @@ -0,0 +1,1129 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/template" +) + +const ( + // kubernetesCondMode records on the RestoreJob whether it rebuilds a + // deleted cluster or rolls a live one back (Reason), and which cluster + // (Message). It lives in status, which the creator of a RestoreJob cannot + // set, so a RestoreJob cannot claim a live cluster is gone. + kubernetesCondMode = "KubernetesRestoreMode" + kubernetesRestorePhaseLabel = "kubernetes.strategy.backups.cozystack.io/restore-phase" + + // What a restore changes on a live cluster is recorded on the objects it + // changes rather than on the RestoreJob, so the cluster can be put back + // after the RestoreJob is deleted, and a retry after a failed restore + // keeps the counts from before the first one rather than the zeros. + kubernetesReplicasBeforeAnnotation = "kubernetes.strategy.backups.cozystack.io/replicas-before-restore" + kubernetesSuspendedByAnnotation = "kubernetes.strategy.backups.cozystack.io/suspended-by-restore" + kubernetesReplicasUnset = "unset" + + kubernetesRestoreModeRecreate = "recreate" + kubernetesRestoreModeInPlace = "in-place" + + kubernetesPhaseSecrets = "secrets" + kubernetesPhaseVolumes = "volumes" + + machineExcludeDrainAnnotation = "machine.cluster.x-k8s.io/exclude-node-draining" + machineExcludeVolumeDetachAnnotation = "machine.cluster.x-k8s.io/exclude-wait-for-node-volume-detach" + + kubernetesCondVerified = "KubernetesBackupVerified" + kubernetesCondRecreated = "KubernetesClusterRecreated" + kubernetesCondStopped = "KubernetesClusterStopped" + kubernetesCondSecretsRestored = "KubernetesSecretsRestored" + kubernetesCondEtcdRestored = "KubernetesEtcdRestored" + kubernetesCondVolumesRestored = "KubernetesVolumesRestored" + // kubernetesCondEtcdWriteFailed marks a restore whose etcd Job failed + // after it started replacing the prefix. The control plane is then left + // stopped, by a failure and by the RestoreJob's deletion alike. + kubernetesCondEtcdWriteFailed = "KubernetesEtcdWriteFailed" + // kubernetesCondEtcdJobStarted records that a container of the restore + // Job was seen running, which is the only trace left once the Job's + // deadline has deleted its Pods. + kubernetesCondEtcdJobStarted = "KubernetesEtcdJobStarted" + + // The restore Job writes this to its termination message just before it + // deletes the prefix, so a failed Job tells whether the prefix is still + // intact. + kubernetesJobWriting = "writing" +) + +// KubernetesRestoreOptions is the typed shape of RestoreJob.Spec.Options for +// the Kubernetes driver. Both halves of the backup are restored unless one is +// switched off. +type KubernetesRestoreOptions struct { + // Etcd restores the cluster's etcd prefix (its API objects). + Etcd *bool `json:"etcd,omitempty"` + // Volumes restores the DataVolumes backing the cluster's PersistentVolumes. + Volumes *bool `json:"volumes,omitempty"` +} + +func (o KubernetesRestoreOptions) restoreEtcd() bool { return boolDefault(o.Etcd, true) } +func (o KubernetesRestoreOptions) restoreVolumes() bool { return boolDefault(o.Volumes, true) } + +func parseKubernetesRestoreOptions(opts *runtime.RawExtension) (KubernetesRestoreOptions, error) { + var out KubernetesRestoreOptions + if opts == nil || len(opts.Raw) == 0 { + return out, nil + } + if err := json.Unmarshal(opts.Raw, &out); err != nil { + return KubernetesRestoreOptions{}, fmt.Errorf("decode restoreJob.spec.options: %w", err) + } + if !out.restoreEtcd() && !out.restoreVolumes() { + return KubernetesRestoreOptions{}, fmt.Errorf("restoreJob.spec.options disables both etcd and volumes; nothing to restore") + } + return out, nil +} + +type kubernetesRestoreState struct { + Mode string `json:"mode"` + Release string `json:"release"` +} + +func kubernetesAutoscalerName(release string) string { + return release + "-cluster-autoscaler" +} + +func kubernetesBackupDataVolumes(b *backupsv1alpha1.Backup) []string { + raw := b.Spec.DriverMetadata[kubernetesDataVolumesMetaKey] + if raw == "" { + return nil + } + return strings.Split(raw, ",") +} + +func kubernetesBackupHelmReleases(b *backupsv1alpha1.Backup) ([]map[string]any, error) { + var out []map[string]any + if err := json.Unmarshal([]byte(b.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey]), &out); err != nil { + return nil, fmt.Errorf("decode recorded HelmReleases: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("the Backup records no HelmReleases") + } + return out, nil +} + +// kubernetesPreRestoreKey is where the restore Job keeps the prefix it is +// about to replace. The Job writes it only when none is there and deletes it +// once the restore succeeds, so after a failed restore it holds the prefix +// as it was before the first failed attempt, and a retry does not overwrite +// it with a partial one. It is fixed per backup, so the Backup's cleanup +// removes it. +func kubernetesPreRestoreKey(objectKey string) string { + return objectKey + ".pre-restore" +} + +// reconcileKubernetesRestore restores a tenant Kubernetes application in +// place, under its own name and namespace. +// +// When the application is gone, the restore rebuilds it: control-plane PKI +// Secrets first (the etcd Job authenticates with one of them), then the etcd +// prefix, the volumes, and finally the HelmReleases, so the control plane +// comes up on restored state and the existing CA. +// +// When the application is live, the restore stops what the selected halves +// need stopped — the workers for volumes, so the volumes are detached, then +// the control plane for etcd, so nothing writes while the prefix is replaced +// — writes the state back, and returns the cluster to the replica counts it +// had. +// +// Nothing is changed before the backup's artifacts are confirmed to exist. +func (r *RestoreJobReconciler) reconcileKubernetesRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || + rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + return ctrl.Result{}, nil + } + if err := validateKubernetesApplicationRef(backup.Spec.ApplicationRef); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + appName := backup.Spec.ApplicationRef.Name + if t := rj.Spec.TargetApplicationRef; t != nil && ((t.Name != "" && t.Name != appName) || (t.Kind != "" && t.Kind != kubernetesAppKind)) { + return r.markRestoreJobFailed(ctx, rj, "the Kubernetes strategy restores only into the application the backup was taken from") + } + if rj.Namespace != backup.Namespace { + return r.markRestoreJobFailed(ctx, rj, "the Kubernetes strategy restores only into the namespace the backup was taken from") + } + opts, err := parseKubernetesRestoreOptions(rj.Spec.Options) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + + if rj.Status.StartedAt == nil { + base := rj.DeepCopy() + now := metav1.Now() + rj.Status.StartedAt = &now + rj.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning + if err := r.Status().Patch(ctx, rj, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.requeueRestoreStrategyNotReady(ctx, rj, backup.Spec.StrategyRef.Name) + } + return ctrl.Result{}, err + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVerified) { + done, msg, err := r.verifyKubernetesBackup(ctx, rj, backup, strategy, opts) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVerified, "the backup's artifacts are present") + } + + state := kubernetesStateFromRestoreJob(rj) + if state == nil { + return r.captureKubernetesRestoreState(ctx, rj, backup) + } + + if state.Mode == kubernetesRestoreModeInPlace { + return r.reconcileKubernetesInPlace(ctx, rj, backup, strategy, opts, state) + } + return r.reconcileKubernetesRecreate(ctx, rj, backup, strategy, opts, state) +} + +func kubernetesStateFromRestoreJob(rj *backupsv1alpha1.RestoreJob) *kubernetesRestoreState { + c := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondMode) + if c == nil { + return nil + } + mode := kubernetesRestoreModeInPlace + if c.Reason == "Recreate" { + mode = kubernetesRestoreModeRecreate + } + return &kubernetesRestoreState{Mode: mode, Release: c.Message} +} + +// verifyKubernetesBackup confirms, before anything is stopped or deleted, +// that the Velero backup completed and still exists, and that the etcd dump +// object is in the bucket. The dump is checked by a verify-mode run of the +// strategy Job, the only component holding the bucket credentials. +func (r *RestoreJobReconciler) verifyKubernetesBackup(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions) (bool, string, error) { + name := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if name == "" { + return false, "the Backup records no Velero backup", nil + } + vb := &velerov1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb); err != nil { + if apierrors.IsNotFound(err) { + return false, fmt.Sprintf("Velero backup %s/%s no longer exists", veleroNamespace, name), nil + } + return false, "", err + } + if vb.Status.Phase != velerov1.BackupPhaseCompleted { + return false, fmt.Sprintf("Velero backup %s/%s is %q, not Completed", veleroNamespace, name, vb.Status.Phase), nil + } + if !opts.restoreEtcd() { + return true, "", nil + } + return r.runKubernetesJob(ctx, rj, backup, strategy, kubernetesModeVerify, rj.Name+"-verify", kubernetesBackupDatastore(backup)) +} + +// captureKubernetesRestoreState decides between rebuilding a deleted cluster +// and restoring a live one. +func (r *RestoreJobReconciler) captureKubernetesRestoreState(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + release := kubernetesReleaseName(backup.Spec.ApplicationRef.Name) + reason := "Recreate" + _, err := r.Resource(helmReleaseGVR).Namespace(rj.Namespace).Get(ctx, release, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return ctrl.Result{}, err + default: + reason = "InPlace" + mismatch, err := r.kubernetesClusterMismatch(ctx, rj.Namespace, release, backup) + if err != nil { + return ctrl.Result{}, err + } + if mismatch != "" { + return r.markRestoreJobFailed(ctx, rj, mismatch) + } + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondMode, + Status: metav1.ConditionTrue, + Reason: reason, + Message: release, + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +// capiNotServed reports a list of a Cluster API resource that failed because +// the apiserver does not serve that resource at the version the driver asks +// for: a NotFound on a collection, or no RESTMapper match. +func capiNotServed(err error) bool { + return err != nil && (apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err)) +} + +// kubernetesClusterMismatch reports why the live cluster is not the one the +// backup was taken from, or "" when it is or cannot be told apart. A cluster +// deleted and created again under the same name has a new CA, service-account +// key and Talos secrets, which only a rebuild restores: writing the old prefix +// into it would leave every token and node signed by keys it no longer has. +// Backups that predate the recorded UID fall back to the datastore schema, +// which tells the two apart only on a Kamaji that keys it on the UID. +func (r *RestoreJobReconciler) kubernetesClusterMismatch(ctx context.Context, ns, release string, backup *backupsv1alpha1.Backup) (string, error) { + tcp, err := r.Resource(tenantControlPlaneGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return "", nil + } + if err != nil { + return "", err + } + md := backup.Spec.DriverMetadata + recorded, live := md[kubernetesControlPlaneUIDMetaKey], string(tcp.GetUID()) + if recorded == "" { + recorded = md[kubernetesDatastorePrefixMetaKey] + live, _, _ = unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + } + if recorded == "" || live == "" || recorded == live { + return "", nil + } + return fmt.Sprintf("TenantControlPlane %s/%s is not the cluster this backup was taken from (%s, backup has %s): it was deleted and created again under the same name, so an in-place restore would write the old state over a new CA and service-account key; delete the application and restore again to rebuild it with the backed-up PKI", ns, release, live, recorded), nil +} + +func (r *RestoreJobReconciler) listClusterMachineDeployments(ctx context.Context, ns, release string) ([]unstructured.Unstructured, error) { + list, err := r.Resource(machineDeploymentGVR).Namespace(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + var out []unstructured.Unstructured + for _, md := range list.Items { + if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster == release { + out = append(out, md) + } + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Rebuild a deleted cluster +// --------------------------------------------------------------------------- + +// reconcileKubernetesRecreate rebuilds a deleted cluster. The control-plane +// PKI and the volumes come back first, then the cluster's own HelmRelease. +// Kamaji gives the new TenantControlPlane a fresh datastore schema (keyed on +// its UID), so the etcd half can only be written once that schema exists: +// the restore then stops the new control plane like a live one, writes the +// dump into the new prefix, and starts it again. The worker pools are +// recreated last, so their workers join the restored cluster rather than an +// empty one. +func (r *RestoreJobReconciler) reconcileKubernetesRecreate(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, error) { + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondSecretsRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseSecrets, state.Release) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondSecretsRestored, "control-plane Secrets restored") + } + if opts.restoreVolumes() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVolumesRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseVolumes, state.Release) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVolumesRestored, "volumes restored") + } + + if !opts.restoreEtcd() { + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases, true); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster rebuilt from backup without its etcd state") + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondRecreated) { + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases[:1], false); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + ready, err := r.tenantControlPlaneSchemaReady(ctx, rj.Namespace, state.Release) + if err != nil { + return ctrl.Result{}, err + } + if ready { + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondRecreated, "the control plane is up with a datastore schema") + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondRecreated, + Status: metav1.ConditionFalse, + Reason: "Waiting", + Message: "waiting for TenantControlPlane " + state.Release + " to report its datastore schema", + }) + cond := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondRecreated) + if time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf("TenantControlPlane %s/%s reported no datastore schema within %s", rj.Namespace, state.Release, restorePreparationDeadline)) + } + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + return r.reconcileKubernetesInPlace(ctx, rj, backup, strategy, KubernetesRestoreOptions{Etcd: new(true), Volumes: new(false)}, state) +} + +func (r *RestoreJobReconciler) tenantControlPlaneSchemaReady(ctx context.Context, ns, release string) (bool, error) { + tcp, err := r.Resource(tenantControlPlaneGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + if err != nil { + return false, client.IgnoreNotFound(err) + } + schema, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + return schema != "", nil +} + +// createKubernetesHelmReleases recreates the recorded HelmReleases. A worker +// pool's HelmRelease outlives the deletion of its cluster (KubernetesNodes is +// an application of its own), but its MachineDeployment was garbage-collected +// with the CAPI Cluster, and nothing re-applies an unchanged release. One +// that exists is therefore forced through a Helm upgrade, which recreates +// the objects its release is missing. The cluster's own release is created +// while the rebuild polls for its schema, so it passes force=false: forcing +// it on every poll would re-run its chart while Kamaji brings it up. +func (r *RestoreJobReconciler) createKubernetesHelmReleases(ctx context.Context, ns string, releases []map[string]any, force bool) error { + for _, hr := range releases { + obj := &unstructured.Unstructured{Object: hr} + obj.SetNamespace(ns) + _, err := r.Resource(helmReleaseGVR).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{}) + if err == nil { + continue + } + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("recreate HelmRelease %s/%s: %w", ns, obj.GetName(), err) + } + if !force { + continue + } + now := time.Now().UTC().Format(time.RFC3339Nano) + patch, merr := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "reconcile.fluxcd.io/requestedAt": now, + "reconcile.fluxcd.io/forceAt": now, + }}}) + if merr != nil { + return merr + } + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, obj.GetName(), types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("force HelmRelease %s/%s to reconcile: %w", ns, obj.GetName(), err) + } + } + return nil +} + +// --------------------------------------------------------------------------- +// Restore a live cluster +// --------------------------------------------------------------------------- + +func (r *RestoreJobReconciler) reconcileKubernetesInPlace(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, error) { + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondStopped) { + stopped, waiting, err := r.stopKubernetesCluster(ctx, rj, backup, opts, state.Release) + if capiNotServed(err) { + // Reading this as "no workers" would delete volumes that running + // workers still mount. Workers are listed before anything is + // stopped, so the resume below has only what it already stopped + // to put back. + return r.failKubernetesInPlace(ctx, rj, state, fmt.Sprintf("Cluster API is not served at %s: %v", machineDeploymentGVR.GroupVersion(), err)) + } + if err != nil { + // Retried like any other wait, under the same deadline: a single + // apiserver blip must not abort a restore. + stopped, waiting = false, fmt.Sprintf("a retry after an API error: %v", err) + } + if stopped { + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondStopped, "cluster stopped for restore") + } + // The condition's transition time marks when stopping began: it is + // kept while the status stays False, whatever the message says. + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondStopped, + Status: metav1.ConditionFalse, + Reason: "Stopping", + Message: "waiting for " + waiting, + }) + cond := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped) + if time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline { + return r.failKubernetesInPlace(ctx, rj, state, fmt.Sprintf("the cluster did not stop within %s: still waiting for %s", restorePreparationDeadline, waiting)) + } + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + if res, next, err := r.kubernetesRestoreData(ctx, rj, backup, strategy, opts, state); !next { + return res, err + } + + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil { + return ctrl.Result{}, err + } + if state.Mode == kubernetesRestoreModeRecreate { + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases[1:], true); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster rebuilt from backup") + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster restored in place") +} + +// stopKubernetesCluster suspends the cluster's HelmReleases so Flux does not +// undo the scale-down, then stops the workers when volumes are restored — +// deleting the data DataVolumes and PVCs the backup holds once nothing mounts +// them — and the control plane when etcd is. Workers go first: CAPI removes a +// Machine's Node through the tenant API. It reports whether all of that has +// settled and, if not, what it is waiting for. +func (r *RestoreJobReconciler) stopKubernetesCluster(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, opts KubernetesRestoreOptions, release string) (bool, string, error) { + ns := rj.Namespace + mds, err := r.listClusterMachineDeployments(ctx, ns, release) + if err != nil { + return false, "", err + } + for _, hr := range kubernetesHelmReleaseNames(mds, release) { + if err := r.suspendHelmReleaseForRestore(ctx, ns, hr); err != nil { + return false, "", err + } + } + + if opts.restoreVolumes() { + if err := r.stopReplicas(ctx, deploymentGVR, ns, kubernetesAutoscalerName(release)); err != nil { + return false, "", err + } + for _, md := range mds { + if err := r.stopReplicas(ctx, machineDeploymentGVR, ns, md.GetName()); err != nil { + return false, "", err + } + } + remaining, err := r.releaseClusterMachines(ctx, ns, release) + if err != nil { + return false, "", err + } + if remaining > 0 { + return false, fmt.Sprintf("%d worker Machine(s) to be deleted", remaining), nil + } + var pending []string + for _, name := range kubernetesBackupDataVolumes(backup) { + dvGone, err := r.deleteDataVolume(ctx, ns, name) + if err != nil { + return false, "", err + } + pvcGone, err := r.deletePVC(ctx, ns, name) + if err != nil { + return false, "", err + } + if !dvGone || !pvcGone { + pending = append(pending, name) + } + } + if len(pending) > 0 { + return false, "data volumes to be deleted: " + strings.Join(pending, ", "), nil + } + } + + if opts.restoreEtcd() { + if err := r.stopReplicas(ctx, kamajiControlPlaneGVR, ns, release); err != nil { + return false, "", err + } + cp, err := r.Resource(deploymentGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return false, "", err + default: + if n, _, _ := unstructured.NestedInt64(cp.Object, "status", "replicas"); n > 0 { + return false, fmt.Sprintf("%d control-plane Pod(s) to stop", n), nil + } + // status.replicas leaves out Pods that are still terminating, and + // a terminating kube-apiserver can still write to etcd. + selector, _, _ := unstructured.NestedStringMap(cp.Object, "spec", "selector", "matchLabels") + if len(selector) > 0 { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(ns), client.MatchingLabels(selector)); err != nil { + return false, "", err + } + if len(pods.Items) > 0 { + return false, fmt.Sprintf("%d control-plane Pod(s) to terminate", len(pods.Items)), nil + } + } + } + } + return true, "", nil +} + +// releaseClusterMachines lets the cluster's Machines finish deleting without +// draining their Nodes or waiting for volumes to detach: the workers are +// recreated after the restore, and a drain can block on a PodDisruptionBudget. +// It returns how many Machines remain. +func (r *RestoreJobReconciler) releaseClusterMachines(ctx context.Context, ns, release string) (int, error) { + machines, err := r.Resource(machineGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: clusterNameLabel + "=" + release}) + if err != nil { + return 0, err + } + patch := []byte(`{"metadata":{"annotations":{"` + machineExcludeDrainAnnotation + `":"","` + machineExcludeVolumeDetachAnnotation + `":""}}}`) + for _, m := range machines.Items { + ann := m.GetAnnotations() + if _, ok := ann[machineExcludeDrainAnnotation]; ok { + if _, ok := ann[machineExcludeVolumeDetachAnnotation]; ok { + continue + } + } + if _, err := r.Resource(machineGVR).Namespace(ns).Patch(ctx, m.GetName(), types.MergePatchType, patch, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + return 0, err + } + } + return len(machines.Items), nil +} + +// suspendHelmReleaseForRestore suspends a HelmRelease and marks it as +// suspended by the restore. One already suspended is left alone and unmarked, +// so resuming the cluster does not resume it. +func (r *RestoreJobReconciler) suspendHelmReleaseForRestore(ctx context.Context, ns, name string) error { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + if suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend"); suspended { + return nil + } + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesSuspendedByAnnotation + `":"true"}},"spec":{"suspend":true}}`) + _, err = r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// stopReplicas scales an object to zero, first recording its replica count +// on it. A count already recorded is kept: it was taken before an earlier +// restore stopped the object. +func (r *RestoreJobReconciler) stopReplicas(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error { + obj, err := r.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + annotations := map[string]any{} + if _, recorded := obj.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; !recorded { + before := kubernetesReplicasUnset + if n, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas"); found { + before = strconv.FormatInt(n, 10) + } + annotations[kubernetesReplicasBeforeAnnotation] = before + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": annotations}, + "spec": map[string]any{"replicas": 0}, + }) + if err != nil { + return err + } + _, err = r.Resource(gvr).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// resumeReplicas puts back the replica count stopReplicas recorded — removing +// the field when it was unset, so its owner (the autoscaler, for a +// MachineDeployment) takes it back — and drops the record. +func (r *RestoreJobReconciler) resumeReplicas(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error { + obj, err := r.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + before, recorded := obj.GetAnnotations()[kubernetesReplicasBeforeAnnotation] + if !recorded { + return nil + } + var replicas any + if before != kubernetesReplicasUnset { + n, err := strconv.ParseInt(before, 10, 64) + if err != nil { + return fmt.Errorf("%s %s/%s: bad %s %q", gvr.Resource, ns, name, kubernetesReplicasBeforeAnnotation, before) + } + replicas = n + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{kubernetesReplicasBeforeAnnotation: nil}}, + "spec": map[string]any{"replicas": replicas}, + }) + if err != nil { + return err + } + _, err = r.Resource(gvr).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// resumeKubernetesCluster undoes stopKubernetesCluster from the records on +// the objects themselves, so it needs nothing from the RestoreJob and is safe +// to call more than once. +func (r *RestoreJobReconciler) resumeKubernetesCluster(ctx context.Context, ns, release string) error { + var errs []string + mds, err := r.listClusterMachineDeployments(ctx, ns, release) + if err != nil && !capiNotServed(err) { + return err + } + if err := r.resumeReplicas(ctx, kamajiControlPlaneGVR, ns, release); err != nil { + errs = append(errs, err.Error()) + } + for _, md := range mds { + if err := r.resumeReplicas(ctx, machineDeploymentGVR, ns, md.GetName()); err != nil { + errs = append(errs, err.Error()) + } + } + if err := r.resumeReplicas(ctx, deploymentGVR, ns, kubernetesAutoscalerName(release)); err != nil { + errs = append(errs, err.Error()) + } + for _, name := range kubernetesHelmReleaseNames(mds, release) { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err.Error()) + } + continue + } + if _, marked := hr.GetAnnotations()[kubernetesSuspendedByAnnotation]; !marked { + continue + } + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesSuspendedByAnnotation + `":null}},"spec":{"suspend":false}}`) + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("resume cluster: %s", strings.Join(errs, "; ")) + } + return nil +} + +// failKubernetesInPlace fails a live-cluster restore. The cluster is resumed +// unless the etcd prefix may have been partly replaced: bringing a control +// plane up on that would serve a mix of old and restored objects, so it is +// left stopped for a retry or a manual repair. +func (r *RestoreJobReconciler) failKubernetesInPlace(ctx context.Context, rj *backupsv1alpha1.RestoreJob, state *kubernetesRestoreState, msg string) (ctrl.Result, error) { + if apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdWriteFailed) { + return r.markRestoreJobFailed(ctx, rj, msg) + } + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil { + msg = fmt.Sprintf("%s; the cluster could not be fully resumed: %v", msg, err) + } + return r.markRestoreJobFailed(ctx, rj, msg) +} + +// cleanupKubernetesRestore runs when a RestoreJob is deleted. A live cluster +// it stopped is resumed, unless its etcd prefix may be partly replaced, which +// is left for the next restore to repair and reported as a Warning event. +func (r *RestoreJobReconciler) cleanupKubernetesRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob) { + state := kubernetesStateFromRestoreJob(rj) + if state == nil || state.Release == "" || rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded { + return + } + partial := apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdWriteFailed) + // A restore Job still running may be rewriting the prefix right now; + // one that failed after it started writing left it partial. Either way + // the control plane must not come up on it. The Job is deleted so it + // stops writing; the garbage collector would only get to it later. + job := &batchv1.Job{} + switch err := r.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: jobNameForRestoreJob(rj)}, job); { + case err == nil && jobConditionState(job) != batchv1.JobComplete: + wrote, werr := r.kubernetesJobReachedWrite(ctx, rj, job.Name) + if werr != nil || wrote || jobConditionState(job) != batchv1.JobFailed { + partial = true + } + policy := metav1.DeletePropagationBackground + _ = r.Delete(ctx, job, &client.DeleteOptions{PropagationPolicy: &policy}) + case err != nil && !apierrors.IsNotFound(err): + partial = true + } + if partial { + if r.Recorder != nil { + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "ClusterLeftStopped", + "%s/%s stays stopped: its etcd prefix may be partly restored; run a new restore from a Backup", rj.Namespace, state.Release) + } + return + } + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil && r.Recorder != nil { + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "CleanupFailed", "resume %s/%s: %v", rj.Namespace, state.Release, err) + } +} + +// --------------------------------------------------------------------------- +// Steps shared by both modes +// --------------------------------------------------------------------------- + +// kubernetesRestoreData writes back the selected halves, etcd before volumes. +// next is true when both are done and the caller may move on; otherwise the +// returned result is the reconcile's. +func (r *RestoreJobReconciler) kubernetesRestoreData(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, bool, error) { + if opts.restoreEtcd() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdRestored) { + ds, err := resolveKubernetesDatastore(ctx, r.Interface, rj.Namespace, backup.Spec.ApplicationRef.Name) + if err != nil { + res, rerr := r.kubernetesStepResult(ctx, rj, state, nil, fmt.Sprintf("resolve the cluster's datastore: %v", err), false) + return res, false, rerr + } + ds.SourcePrefix = backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] + done, msg, err := r.runKubernetesJob(ctx, rj, backup, strategy, kubernetesModeRestore, jobNameForRestoreJob(rj), *ds) + if msg != "" && err == nil { + wrote, werr := r.kubernetesJobReachedWrite(ctx, rj, jobNameForRestoreJob(rj)) + if werr != nil { + return ctrl.Result{}, false, werr + } + if !wrote { + res, rerr := r.kubernetesStepResult(ctx, rj, state, nil, "etcd restore failed before it changed the prefix: "+msg, false) + return res, false, rerr + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondEtcdWriteFailed, + Status: metav1.ConditionTrue, + Reason: "RestoreJobFailed", + Message: msg, + }) + msg = fmt.Sprintf("etcd restore failed: %s. The prefix may be partly restored, so the control plane is left stopped. "+ + "A copy of the prefix from before the first restore of this backup that did not finish is kept at %s in the backup bucket; run a new RestoreJob to retry", + msg, kubernetesPreRestoreKey(backup.Spec.DriverMetadata[kubernetesObjectMetaKey])) + } + if err == nil && msg == "" && !done && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdJobStarted) { + started, serr := r.kubernetesJobStarted(ctx, rj.Namespace, jobNameForRestoreJob(rj)) + if serr != nil { + return ctrl.Result{}, false, serr + } + if started { + res, cerr := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondEtcdJobStarted, "the etcd restore Job is running") + return res, false, cerr + } + } + if err != nil || msg != "" || !done { + res, rerr := r.kubernetesStepResult(ctx, rj, state, err, msg, done) + return res, false, rerr + } + res, err := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondEtcdRestored, "etcd prefix restored") + return res, false, err + } + if opts.restoreVolumes() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVolumesRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseVolumes, kubernetesReleaseName(backup.Spec.ApplicationRef.Name)) + if err != nil || msg != "" || !done { + res, rerr := r.kubernetesStepResult(ctx, rj, state, err, msg, done) + return res, false, rerr + } + res, err := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVolumesRestored, "volumes restored") + return res, false, err + } + return ctrl.Result{}, true, nil +} + +// kubernetesStepResult turns a step's outcome into the reconcile result: an +// API error is retried, a failure message fails the restore (resuming a live +// cluster first where that is safe), and an unfinished step is polled. +func (r *RestoreJobReconciler) kubernetesStepResult(ctx context.Context, rj *backupsv1alpha1.RestoreJob, state *kubernetesRestoreState, err error, failure string, done bool) (ctrl.Result, error) { + switch { + case err != nil: + return ctrl.Result{}, err + case failure != "": + // A rebuild stops its new control plane for the etcd half the same + // way a live restore does, so both put back whatever they stopped. + if state != nil { + return r.failKubernetesInPlace(ctx, rj, state, failure) + } + return r.markRestoreJobFailed(ctx, rj, failure) + case !done: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + return ctrl.Result{}, nil +} + +// kubernetesJobReachedWrite reports whether any Pod of the restore Job may +// have changed the prefix: one running now, or one that terminated with the +// marker the script writes just before it deletes the prefix. A container +// that never ran (an image it could not pull) wrote nothing. When the Job's +// Pods are gone (its deadline deletes them), the answer is whether any of +// them was ever seen running. +func (r *RestoreJobReconciler) kubernetesJobReachedWrite(ctx context.Context, rj *backupsv1alpha1.RestoreJob, jobName string) (bool, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(rj.Namespace), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + if len(pods.Items) == 0 { + return apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdJobStarted), nil + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + switch { + case cs.State.Running != nil: + return true, nil + case cs.State.Terminated != nil: + if strings.TrimSpace(cs.State.Terminated.Message) == kubernetesJobWriting { + return true, nil + } + case cs.LastTerminationState.Terminated != nil: + if strings.TrimSpace(cs.LastTerminationState.Terminated.Message) == kubernetesJobWriting { + return true, nil + } + } + } + } + return false, nil +} + +// kubernetesJobStarted reports whether a container of the restore Job has +// run, which is what kubernetesJobReachedWrite falls back on once the Job's +// Pods are gone. +func (r *RestoreJobReconciler) kubernetesJobStarted(ctx context.Context, ns, jobName string) (bool, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(ns), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Running != nil || cs.State.Terminated != nil || cs.LastTerminationState.Terminated != nil { + return true, nil + } + } + } + return false, nil +} + +// runKubernetesJob runs the strategy Job in the given mode under the given +// name against ds. It returns done once the Job completed, or a failure +// message once it failed. +func (r *RestoreJobReconciler) runKubernetesJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, mode, name string, ds kubernetesDatastore) (bool, string, error) { + if ds.Prefix == "" || len(ds.Endpoints) == 0 { + return false, "the Backup records no etcd datastore prefix or endpoints", nil + } + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, rj.Namespace, + mode, backup.Spec.DriverMetadata[kubernetesObjectMetaKey], ds, kubernetesStrategyParameters(backup)) + if err != nil { + return false, fmt.Sprintf("failed to template Kubernetes strategy: %v", err), nil + } + job, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, rj, rj.Namespace, name, + map[string]string{ + kubernetesLabelMode: mode, + backupsv1alpha1.OwningJobNameLabel: rj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace, + }, rendered) + if err != nil { + return false, "", err + } + switch jobConditionState(job) { + case batchv1.JobComplete: + return true, "", nil + case batchv1.JobFailed: + msg := jobFailureMessage(job) + if msg == "" { + msg = fmt.Sprintf("%s Job reported Failed", mode) + } + return false, msg, nil + } + return false, "", nil +} + +// runKubernetesVeleroRestore restores one phase of the Velero backup: +// control-plane Secrets, or the cluster's data DataVolumes and PVCs. It +// returns done once the Velero Restore completed. +func (r *RestoreJobReconciler) runKubernetesVeleroRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, phase, release string) (bool, string, error) { + veleroBackup := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if veleroBackup == "" { + return false, "the Backup records no Velero backup", nil + } + labels := map[string]string{ + backupsv1alpha1.OwningJobNameLabel: rj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace, + kubernetesRestorePhaseLabel: phase, + } + restores := &velerov1.RestoreList{} + if err := r.List(ctx, restores, client.InNamespace(veleroNamespace), client.MatchingLabels(labels)); err != nil { + return false, "", err + } + if len(restores.Items) == 0 { + spec, err := kubernetesVeleroRestoreSpec(strategy, backup, phase, release) + if err != nil { + return false, fmt.Sprintf("failed to build Velero Restore: %v", err), nil + } + spec.BackupName = veleroBackup + cm, err := r.kubernetesResourceModifiers(ctx, rj, phase, labels) + if err != nil { + return false, "", err + } + spec.ResourceModifier = &corev1.TypedLocalObjectReference{APIGroup: new(""), Kind: "ConfigMap", Name: cm.Name} + restore := &velerov1.Restore{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s.%s-%s-", rj.Namespace, rj.Name, phase), + Namespace: veleroNamespace, + Labels: labels, + }, + Spec: *spec, + } + if err := r.Create(ctx, restore); err != nil { + return false, "", err + } + if r.Recorder != nil { + r.Recorder.Event(rj, corev1.EventTypeNormal, "VeleroRestoreCreated", fmt.Sprintf("Created Velero Restore %s/%s", veleroNamespace, restore.Name)) + } + return false, "", nil + } + restore := &restores.Items[0] + switch restore.Status.Phase { + case velerov1.RestorePhaseCompleted: + return true, "", nil + case velerov1.RestorePhaseFailed, velerov1.RestorePhasePartiallyFailed, velerov1.RestorePhaseFailedValidation: + msg := fmt.Sprintf("Velero Restore %s/%s finished %s", restore.Namespace, restore.Name, restore.Status.Phase) + if restore.Status.FailureReason != "" { + msg += ": " + restore.Status.FailureReason + } + return false, msg, nil + } + return false, "", nil +} + +// kubernetesVeleroRestoreSpec starts from the strategy's restore spec and +// narrows it to one phase. The selectors mirror the ones the shipped backup +// spec uses, so a phase restores exactly its share of the backup. +func kubernetesVeleroRestoreSpec(strategy *strategyv1alpha1.Kubernetes, backup *backupsv1alpha1.Backup, phase, release string) (*velerov1.RestoreSpec, error) { + spec := &velerov1.RestoreSpec{} + if strategy.Spec.Velero.RestoreSpec != nil { + rendered, err := template.Template(strategy.Spec.Velero.RestoreSpec, map[string]any{ + "Application": map[string]any{ + "metadata": map[string]any{"name": backup.Spec.ApplicationRef.Name, "namespace": backup.Namespace}, + "kind": backup.Spec.ApplicationRef.Kind, + }, + "Parameters": kubernetesStrategyParameters(backup), + }) + if err != nil { + return nil, err + } + spec = rendered + } + spec.IncludedNamespaces = []string{backup.Namespace} + spec.LabelSelector = nil + switch phase { + case kubernetesPhaseSecrets: + spec.IncludedResources = []string{"secrets"} + spec.OrLabelSelectors = []*metav1.LabelSelector{ + {MatchLabels: map[string]string{kamajiNameLabel: release}}, + {MatchLabels: map[string]string{helmInstanceLabel: release}}, + } + case kubernetesPhaseVolumes: + spec.IncludedResources = []string{"persistentvolumeclaims", "datavolumes.cdi.kubevirt.io"} + spec.OrLabelSelectors = []*metav1.LabelSelector{ + {MatchLabels: map[string]string{clusterNameLabel: release}}, + } + spec.RestorePVs = new(true) + default: + return nil, fmt.Errorf("unknown restore phase %q", phase) + } + return spec, nil +} + +// kubernetesResourceModifiers creates the ConfigMap of resource modifiers for +// a phase. Restored Secrets drop their ownerReferences, which point at +// objects of the deleted cluster and would let the garbage collector delete +// them again; restored PVCs and DataVolumes let CDI adopt the restored claim. +func (r *RestoreJobReconciler) kubernetesResourceModifiers(ctx context.Context, rj *backupsv1alpha1.RestoreJob, phase string, labels map[string]string) (*corev1.ConfigMap, error) { + var rules []resourceModifierRule + switch phase { + case kubernetesPhaseSecrets: + patch, err := marshalPatchData(map[string]any{"metadata": map[string]any{"ownerReferences": nil}}) + if err != nil { + return nil, err + } + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{GroupResource: "secrets", ResourceNameRegex: ".*", Namespaces: []string{rj.Namespace}}, + MergePatches: []mergePatch{{PatchData: patch}}, + }) + case kubernetesPhaseVolumes: + patch, err := marshalPatchData(map[string]any{"metadata": map[string]any{"annotations": map[string]string{cdiAllowClaimAdoption: "true"}}}) + if err != nil { + return nil, err + } + for _, gr := range []string{"persistentvolumeclaims", "datavolumes.cdi.kubevirt.io"} { + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{GroupResource: gr, ResourceNameRegex: ".*", Namespaces: []string{rj.Namespace}}, + MergePatches: []mergePatch{{PatchData: patch}}, + }) + } + } + data, err := yaml.Marshal(resourceModifiers{Version: "v1", ResourceModifierRules: rules}) + if err != nil { + return nil, err + } + name := fmt.Sprintf("restore-modifiers-%s-%s-%s", rj.Namespace, rj.Name, phase) + if len(name) > 253 { + name = name[:253] + } + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: veleroNamespace, Labels: labels}, + Data: map[string]string{"resource-modifier-rules.yaml": string(data)}, + } + if err := r.Create(ctx, cm); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + existing := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Namespace: veleroNamespace, Name: name}, existing); err != nil { + return nil, err + } + existing.Data = cm.Data + if err := r.Update(ctx, existing); err != nil { + return nil, err + } + return existing, nil + } + return cm, nil +} + +func (r *RestoreJobReconciler) setKubernetesRestoreCondition(ctx context.Context, rj *backupsv1alpha1.RestoreJob, condType, message string) (ctrl.Result, error) { + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: condType, + Status: metav1.ConditionTrue, + Reason: "Done", + Message: message, + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +func (r *RestoreJobReconciler) markKubernetesRestoreSucceeded(ctx context.Context, rj *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) { + now := metav1.Now() + rj.Status.CompletedAt = &now + rj.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "RestoreCompleted", + Message: message, + }) + return ctrl.Result{}, r.Status().Update(ctx, rj) +} diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a61f0800..1a4dd7943 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -204,6 +204,8 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) return r.reconcileRedisRestore(ctx, restoreJob, backup) case strategyv1alpha1.KafkaStrategyKind: return r.reconcileKafkaRestore(ctx, restoreJob, backup) + case strategyv1alpha1.KubernetesStrategyKind: + return r.reconcileKubernetesRestore(ctx, restoreJob, backup) default: return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind)) } @@ -382,6 +384,12 @@ func (r *RestoreJobReconciler) cleanupOnDelete(ctx context.Context, restoreJob * case strategyv1alpha1.VeleroStrategyKind: r.cleanupVeleroRestore(ctx, restoreJob) + case strategyv1alpha1.KubernetesStrategyKind: + // Its Velero Restores are labelled like the Velero driver's, so the + // same cleanup reaps them; a live cluster it stopped is resumed. + r.cleanupVeleroRestore(ctx, restoreJob) + r.cleanupKubernetesRestore(ctx, restoreJob) + case strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.JobStrategyKind, strategyv1alpha1.AltinityStrategyKind, strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MongoDBStrategyKind, strategyv1alpha1.FoundationDBStrategyKind, strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind: // Nothing to clean up: these drivers don't materialise namespaced // artifacts that outlive the RestoreJob. (Etcd: the operator-side diff --git a/packages/system/backupstrategy-controller/Makefile b/packages/system/backupstrategy-controller/Makefile index 94b1931d9..aabc11d17 100644 --- a/packages/system/backupstrategy-controller/Makefile +++ b/packages/system/backupstrategy-controller/Makefile @@ -6,9 +6,9 @@ include ../../../hack/package.mk # PHONY so a stray file/dir named `image` cannot mark the aggregate up to date # and skip both builds. -.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client +.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup -image: image-backupstrategy-controller image-redis-backup image-backup-http-client +image: image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup image-backupstrategy-controller: docker buildx build -f images/backupstrategy-controller/Dockerfile ../../.. \ @@ -46,5 +46,16 @@ image-backup-http-client: image-redis-backup yq -i '.backupStrategyController.chBackupClientImage = strenv(IMAGE) | .backupStrategyController.rabbitmqBackupClientImage = strenv(IMAGE)' values.yaml rm -f images/backup-http-client.json +# Chained after image-backup-http-client: one `yq -i` writer at a time. +image-kubernetes-backup: image-backup-http-client + docker buildx build -f images/kubernetes-backup/Dockerfile images/kubernetes-backup \ + $(call image-tags,kubernetes-backup,v$(COZYSTACK_VERSION)) \ + $(call cache-args,kubernetes-backup) \ + --metadata-file images/kubernetes-backup.json \ + $(BUILDX_ARGS) + IMAGE="$(REGISTRY)/kubernetes-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubernetes-backup.json -o json -r)" \ + yq -i '.backupStrategyController.kubernetesBackupClientImage = strenv(IMAGE)' values.yaml + rm -f images/kubernetes-backup.json + test: helm unittest . diff --git a/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml b/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml new file mode 100644 index 000000000..0b681c5be --- /dev/null +++ b/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml @@ -0,0 +1,147 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: kubernetes.strategy.backups.cozystack.io +spec: + group: strategy.backups.cozystack.io + names: + kind: Kubernetes + listKind: KubernetesList + plural: kubernetes + singular: kubernetes + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Kubernetes defines a backup strategy for a tenant Kubernetes application. + A tenant cluster keeps its state in two places the platform can reach: the + cluster's key prefix in the tenant's shared etcd DataStore, and host-side + objects (control-plane PKI Secrets, the application HelmReleases, and the + DataVolumes that back the tenant's PersistentVolumes). The strategy captures + the first with a one-shot Job and the second with Velero. The two are taken + one after another, so a backup is crash-consistent at best. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: KubernetesSpec specifies the desired behavior of a Kubernetes + backup strategy. + properties: + certificateWarningThreshold: + description: |- + CertificateWarningThreshold is how close to expiry a control-plane + certificate may be before a backup records a warning. Defaults to 720h. + type: string + template: + description: |- + Template holds a PodTemplateSpec for the Job that dumps the cluster's + key prefix from its etcd DataStore to object storage, writes it back on + restore, and deletes the object on cleanup. The Job runs in the + application namespace. Helm-like Go templates are supported in every + string field and are rendered against: + - `.Release.Name` and `.Release.Namespace` — the application's name and + namespace; `.Release.Prefix` is `kubernetes-`. + - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`. + - `.ObjectKey` — the object-storage key of this backup's dump. + - `.Datastore.Prefix` — the cluster's key prefix, without slashes. + - `.Datastore.Endpoints` — comma-separated etcd client URLs. + - `.Datastore.CertificateSecret` — the Secret holding the cluster's + etcd client certificate (`ca.crt`, `server.crt`, `server.key`). + - `.Parameters` — the resolved BackupClass strategy parameters. + + Stored schemaless for the same CRD-size reason as redis_types.go. + x-kubernetes-preserve-unknown-fields: true + velero: + description: |- + Velero holds the Velero Backup and Restore specs for the host-side + objects and volumes. Rendered against `.Application` and `.Parameters`, + like the Velero strategy. + x-kubernetes-preserve-unknown-fields: true + required: + - template + - velero + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile b/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile new file mode 100644 index 000000000..b897e51be --- /dev/null +++ b/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile @@ -0,0 +1,14 @@ +# kubernetes-backup runs the Kubernetes strategy Pod: it dumps a tenant +# cluster's key prefix from its etcd DataStore with etcdctl, moves the dump to +# S3 with curl --aws-sigv4, and writes it back with etcdctl on restore. etcdctl +# matches the etcd the platform deploys. Everything is installed at build time +# so the Pods never run `apk add` and work on air-gapped clusters. +FROM quay.io/coreos/etcd:v3.6.11@sha256:6ae247c7666ceec554c51ba1f9bc8dd2212f975370dbd65710c9ca0e36ae1fff AS etcd + +FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 + +RUN apk add --no-cache curl jq ca-certificates coreutils + +COPY --from=etcd /usr/local/bin/etcdctl /usr/local/bin/etcdctl + +USER 65532:65532 diff --git a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml index 3f3b271e3..347c704d1 100644 --- a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml +++ b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml @@ -78,6 +78,13 @@ spec: apiGroup: strategy.backups.cozystack.io kind: Redis name: cozy-default-redis + - application: + apiGroup: apps.cozystack.io + kind: Kubernetes + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kubernetes + name: cozy-default-kubernetes - application: apiGroup: apps.cozystack.io kind: Kafka diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 8d90980a8..735e16308 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -62,7 +62,25 @@ rules: # through the dynamic client, so no list/watch) - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] - verbs: ["get", "delete"] + verbs: ["get", "list", "delete"] +# Kubernetes strategy: read where a tenant cluster keeps its etcd prefix, and +# stop and resume its control plane, worker pools and cluster-autoscaler around +# an in-place restore. +- apiGroups: ["kamaji.clastix.io"] + resources: ["tenantcontrolplanes", "datastores"] + verbs: ["get"] +- apiGroups: ["controlplane.cluster.x-k8s.io"] + resources: ["kamajicontrolplanes"] + verbs: ["get", "patch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machinedeployments"] + verbs: ["get", "list", "patch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machines"] + verbs: ["list", "patch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "patch"] # HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR # and deletes old. patch additionally lets the DefaultObjectsGate stamp # reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to diff --git a/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml new file mode 100644 index 000000000..3fab192ad --- /dev/null +++ b/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml @@ -0,0 +1,304 @@ +{{/* + Two template layers, as in strategy-redis-default.yaml: Helm renders this + file, then the strategy controller renders the resulting CR per BackupJob. + Literal "{{ .Foo }}" for the second layer goes through printf. + + Gated on both a Velero BSL and a resolved $bucketName: the Velero half of + the backup goes to the BSL and the etcd dump to the bucket. Also gated on + its CRD being established, for the upgrade of an existing install that + first ships this kind: see strategy-mongodb-default.yaml. +*/}} +{{- $bucketName := include "backupstrategy-controller.bucketName" . -}} +{{- $crdEstablished := include "backupstrategy-controller.crdEstablished" (dict "root" . "crd" "kubernetes.strategy.backups.cozystack.io") -}} +{{- if and .Values.velero.bslEnabled $bucketName $crdEstablished -}} +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Kubernetes +metadata: + name: cozy-default-kubernetes + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} +spec: + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: {{ int .Values.backupStrategyController.kubernetesBackupActiveDeadlineSeconds }} + volumes: + {{- /* + The cluster's own etcd client certificate, the one its kube-apiserver + uses. Optional so a missing Secret fails the script with a clear cause + instead of wedging the Pod on FailedMount. + */}} + - name: etcd-client + secret: + secretName: {{ printf "{{ .Datastore.CertificateSecret }}" | quote }} + optional: true + {{- if .Values.backupStorage.endpointCASecretName }} + - name: s3-ca + secret: + secretName: {{ .Values.backupStorage.endpointCASecretName | quote }} + optional: true + {{- end }} + containers: + - name: kubernetes-backup + image: "{{ .Values.backupStrategyController.kubernetesBackupClientImage }}" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etcd-client + mountPath: /etc/etcd-client + readOnly: true + {{- if .Values.backupStorage.endpointCASecretName }} + - name: s3-ca + mountPath: /etc/s3-ca + readOnly: true + {{- end }} + env: + - name: ETCDCTL_ENDPOINTS + value: {{ printf "{{ .Datastore.Endpoints }}" | quote }} + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: endpoint + optional: true + - name: S3_BUCKET + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: bucketName + optional: true + - name: S3_REGION + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: region + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: AWS_SECRET_ACCESS_KEY + optional: true + - name: S3_INSECURE_SKIP_VERIFY + value: {{ .Values.backupStorage.insecureSkipTLSVerify | quote }} + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + : "${S3_ENDPOINT:?missing endpoint in cozy-backups-creds}" + : "${S3_BUCKET:?missing bucketName in cozy-backups-creds}" + : "${S3_REGION:?missing region in cozy-backups-creds}" + : "${AWS_ACCESS_KEY_ID:?missing AWS_ACCESS_KEY_ID in cozy-backups-creds}" + : "${AWS_SECRET_ACCESS_KEY:?missing AWS_SECRET_ACCESS_KEY in cozy-backups-creds}" + + MODE="{{ printf "{{ .Mode }}" }}" + KEY="{{ printf "{{ .ObjectKey }}" }}" + SCHEMA="{{ printf "{{ .Datastore.Prefix }}" }}" + SOURCE_SCHEMA="{{ printf "{{ .Datastore.SourcePrefix }}" }}" + + case "${S3_ENDPOINT}" in + http://*|https://*) BASE="${S3_ENDPOINT}" ;; + *) BASE="https://${S3_ENDPOINT}" ;; + esac + TLS="" + case "${BASE}" in + https://*) + if [ -f /etc/s3-ca/ca.crt ]; then TLS="--cacert /etc/s3-ca/ca.crt"; + elif [ -f /etc/s3-ca/tls.crt ]; then TLS="--cacert /etc/s3-ca/tls.crt"; + elif [ "${S3_INSECURE_SKIP_VERIFY:-false}" = "true" ]; then TLS="-k"; + fi + ;; + esac + s3() { + curl -fsS ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@" + } + OBJ="${BASE}/${S3_BUCKET}/${KEY}" + {{- /* + Where a restore keeps the prefix it replaces. One per backup, so + cleanup removes it with the dump. A restore writes it only when it + is absent and deletes it on success, so after a failed restore it + holds the prefix from before the first failed attempt and a retry + cannot overwrite it with a partial one. + */}} + PRE_KEY="${KEY}.pre-restore" + PRE_OBJ="${BASE}/${S3_BUCKET}/${PRE_KEY}" + + if [ "${MODE}" = "cleanup" ]; then + for o in "${OBJ}" "${PRE_OBJ}"; do + echo "deleting ${o#"${BASE}/"}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${o}") + case "${code}" in + 2??|404) echo "deleted (HTTP ${code})" ;; + *) echo "delete failed: HTTP ${code}" >&2; exit 1 ;; + esac + done + exit 0 + fi + + if [ "${MODE}" = "verify" ]; then + s3 -I "${OBJ}" > /dev/null + echo "s3://${S3_BUCKET}/${KEY} exists" + exit 0 + fi + + {{- /* + The prefix comes from the TenantControlPlane: Kamaji's schema is the + TenantControlPlane's UID, or _ for one created by + an older Kamaji. Refuse anything else: an empty or slash-bearing + value would turn the restore's `del --prefix` into a wipe of every + cluster in the shared etcd. + */}} + for s in "${SCHEMA}" "${SOURCE_SCHEMA}"; do + case "${s}" in + ""|*[!a-z0-9_-]*) echo "refusing unexpected datastore prefix '${s}'" >&2; exit 1 ;; + esac + done + PREFIX="/${SCHEMA}/" + {{- /* + The prefix the dump was taken under. A rebuilt cluster has a new + TenantControlPlane and so a new schema; its keys are written under + the new prefix. + */}} + SOURCE_PREFIX="/${SOURCE_SCHEMA}/" + for f in ca.crt server.crt server.key; do + [ -f "/etc/etcd-client/${f}" ] || { echo "etcd client certificate is missing /etc/etcd-client/${f}" >&2; exit 1; } + done + export ETCDCTL_CACERT=/etc/etcd-client/ca.crt + export ETCDCTL_CERT=/etc/etcd-client/server.crt + export ETCDCTL_KEY=/etc/etcd-client/server.key + + if [ "${MODE}" = "backup" ]; then + etcdctl get "${PREFIX}" --prefix -w json > /tmp/dump.json + count=$(jq '.kvs // [] | length' /tmp/dump.json) + if [ "${count}" -eq 0 ]; then + echo "etcd prefix ${PREFIX} is empty" >&2 + exit 1 + fi + echo "dumped ${count} keys under ${PREFIX}; uploading s3://${S3_BUCKET}/${KEY}" + s3 -X PUT --upload-file /tmp/dump.json "${OBJ}" + elif [ "${MODE}" = "restore" ]; then + {{- /* + Everything that can fail without touching etcd runs before the + prefix is deleted: the download, the check that every key belongs + to this cluster, and a copy of the prefix as it is now, which is + what an operator restores from if a failure comes after the delete. + */}} + echo "downloading s3://${S3_BUCKET}/${KEY}" + s3 -o /tmp/dump.json "${OBJ}" + jq -e '.kvs | length > 0' /tmp/dump.json >/dev/null || { echo "dump holds no keys" >&2; exit 1; } + jq -e --arg p "${SOURCE_PREFIX}" 'all(.kvs[]; .key | @base64d | startswith($p))' /tmp/dump.json >/dev/null \ + || { echo "dump holds keys outside ${SOURCE_PREFIX}" >&2; exit 1; } + {{- /* + etcdctl put cannot write an empty value, so a dump holding one + must stop here rather than halfway through the write. + */}} + jq -e 'all(.kvs[]; (.value // "") != "")' /tmp/dump.json >/dev/null \ + || { echo "dump holds a key with an empty value, which etcdctl cannot write" >&2; exit 1; } + jq -r '.kvs[] | [.key, .value] | @tsv' /tmp/dump.json > /tmp/kvs.tsv + {{- /* + Only a definite answer decides about the copy: an existing one is + kept, a missing one is taken, and any other reply (a 5xx, a + timeout) stops the restore while the prefix is still intact, + rather than overwriting the copy an earlier failed restore left. + */}} + code=$(curl -sS -o /dev/null -w '%{http_code}' -I ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + "${PRE_OBJ}") || code=000 + case "${code}" in + 200) echo "keeping s3://${S3_BUCKET}/${PRE_KEY} from an earlier failed restore" ;; + 404) + etcdctl get "${PREFIX}" --prefix -w json > /tmp/before.json + echo "saving the current prefix to s3://${S3_BUCKET}/${PRE_KEY}" + s3 -X PUT --upload-file /tmp/before.json "${PRE_OBJ}" + ;; + *) echo "cannot tell whether s3://${S3_BUCKET}/${PRE_KEY} exists: HTTP ${code}" >&2; exit 1 ;; + esac + {{- /* + Delete first so keys created after the backup do not survive the + restore. Events and master leases are bound to etcd leases that + did not survive; written back without one they would never expire, + so they are skipped. They are matched as whole top-level resources + under the prefix, not as a path segment anywhere, which would also + drop every object in a namespace named "events". + */}} + {{- /* + The termination message tells the driver that this Pod got as far + as changing the prefix: only then is the control plane left + stopped after a failure. + */}} + echo writing > /dev/termination-log + etcdctl del "${PREFIX}" --prefix + n=0 + while IFS="$(printf '\t')" read -r k v; do + key=$(printf '%s' "${k}" | base64 -d) + rel="${key#"${SOURCE_PREFIX}"}" + case "${rel}" in events/*|masterleases/*) continue ;; esac + printf '%s' "${v}" | base64 -d > /tmp/value + etcdctl put "${PREFIX}${rel}" < /tmp/value > /dev/null + n=$((n + 1)) + done < /tmp/kvs.tsv + echo "restored ${n} keys under ${PREFIX}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${PRE_OBJ}") || code=000 + case "${code}" in + 2??|404) echo "dropped s3://${S3_BUCKET}/${PRE_KEY}" ;; + *) echo "WARNING: could not drop s3://${S3_BUCKET}/${PRE_KEY} (HTTP ${code}); a later restore would keep it as the pre-restore copy, so delete it by hand" >&2 ;; + esac + else + echo "unknown mode: ${MODE}" >&2 + exit 1 + fi + resources: + {{- toYaml .Values.backupStrategyController.kubernetesBackupResources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + velero: + {{- /* + Secrets (control-plane and Talos PKI, by the Kamaji and Helm instance + labels) and the DataVolumes and PVCs backing the cluster's + PersistentVolumes (the CSI driver stamps the cluster-name label on them; + worker system disks do not carry it). The HelmReleases are recorded by the + driver on the Backup instead: selecting them by application label would + also select worker disks the lineage webhook labels the same way. The + driver sets the restore's resources and selectors per phase. + */}} + spec: + includedNamespaces: + - {{ printf "{{ .Application.metadata.namespace }}" | quote }} + orLabelSelectors: + - matchLabels: + kamaji.clastix.io/name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + - matchLabels: + app.kubernetes.io/instance: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + - matchLabels: + cluster.x-k8s.io/cluster-name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + includedResources: + - secrets + - persistentvolumeclaims + - datavolumes.cdi.kubevirt.io + includeClusterResources: false + storageLocation: cozy-default + snapshotVolumes: true + snapshotMoveData: true + ttl: 720h0m0s + itemOperationTimeout: 24h0m0s + restoreSpec: + includeClusterResources: false +{{- end -}} diff --git a/packages/system/backupstrategy-controller/tests/rendering_test.yaml b/packages/system/backupstrategy-controller/tests/rendering_test.yaml index 3db6adb14..941444070 100644 --- a/packages/system/backupstrategy-controller/tests/rendering_test.yaml +++ b/packages/system/backupstrategy-controller/tests/rendering_test.yaml @@ -22,6 +22,7 @@ templates: - templates/strategy-foundationdb-default.yaml - templates/strategy-velero-vminstance-default.yaml - templates/strategy-velero-vmdisk-default.yaml + - templates/strategy-kubernetes-default.yaml release: name: t @@ -88,11 +89,14 @@ tests: - hasDocuments: count: 0 template: templates/strategy-velero-vmdisk-default.yaml + - hasDocuments: + count: 0 + template: templates/strategy-kubernetes-default.yaml - hasDocuments: count: 0 template: templates/velero-bsl.yaml - - it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all eleven routes (FoundationDB intentionally unbound)" + - it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all twelve routes (FoundationDB intentionally unbound)" asserts: - hasDocuments: count: 1 @@ -103,7 +107,18 @@ tests: template: templates/backupclass-default.yaml - lengthEqual: path: spec.strategies - count: 11 + count: 12 + template: templates/backupclass-default.yaml + - contains: + path: spec.strategies + content: + application: + apiGroup: apps.cozystack.io + kind: Kubernetes + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kubernetes + name: cozy-default-kubernetes template: templates/backupclass-default.yaml - contains: path: spec.strategies @@ -240,6 +255,18 @@ tests: - hasDocuments: count: 1 template: templates/strategy-velero-vmdisk-default.yaml + - hasDocuments: + count: 1 + template: templates/strategy-kubernetes-default.yaml + # The driver renders these per BackupJob; Helm must leave them literal. + - equal: + path: spec.template.spec.volumes[0].secret.secretName + value: "{{ .Datastore.CertificateSecret }}" + template: templates/strategy-kubernetes-default.yaml + - equal: + path: spec.velero.spec.orLabelSelectors[2].matchLabels["cluster.x-k8s.io/cluster-name"] + value: "kubernetes-{{ .Application.metadata.name }}" + template: templates/strategy-kubernetes-default.yaml - hasDocuments: count: 1 template: templates/velero-bsl.yaml @@ -347,6 +374,7 @@ tests: - {hasDocuments: {count: 0}, template: templates/strategy-foundationdb-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml} # The ungated BackupClass keeps rendering, which is what makes the counts # above an assertion about the CRD gate rather than about a chart-wide # render failure. @@ -432,6 +460,7 @@ tests: - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} + - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} @@ -449,6 +478,7 @@ tests: - {hasDocuments: {count: 0}, template: templates/strategy-redis-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml} - it: "with every CRD Established and the BucketClaim resolved, every Strategy CR renders against the COSI bucket name" # The positive half of the case above, through the same live lookups and no @@ -471,6 +501,7 @@ tests: - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} + - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} @@ -489,6 +520,7 @@ tests: - {hasDocuments: {count: 1}, template: templates/strategy-redis-default.yaml} - {hasDocuments: {count: 1}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 1}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 1}, template: templates/strategy-kubernetes-default.yaml} - it: "velero.bslEnabled=false: Velero strategies and BSL drop out while other strategies keep rendering" set: diff --git a/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml index b13b88173..fa3406d58 100644 --- a/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml +++ b/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml @@ -13,11 +13,11 @@ release: namespace: cozy-backup-controller tests: - - it: "ClusterRole can read and delete DataVolumes" + - it: "ClusterRole can read, list and delete DataVolumes" asserts: - contains: path: rules content: apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] - verbs: ["get", "delete"] + verbs: ["get", "list", "delete"] diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 590d7aca2..121b682d8 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -37,6 +37,24 @@ backupStrategyController: cpu: "1" memory: 1Gi ephemeral-storage: 4Gi + # kubernetesBackupClientImage is rendered into the Kubernetes strategy Pod, + # which dumps a tenant cluster's etcd prefix with etcdctl and moves it to S3 + # with curl. Built from images/kubernetes-backup/; `make image` re-digests it. + kubernetesBackupClientImage: "ghcr.io/cozystack/cozystack/kubernetes-backup:v1.6.0" + # kubernetesBackupActiveDeadlineSeconds bounds one etcd dump/restore Job. + kubernetesBackupActiveDeadlineSeconds: 3600 + # kubernetesBackupResources sizes the Kubernetes strategy Pod. The dump is held + # in ephemeral storage while it moves and jq reads it into memory, so a + # cluster with a large etcd prefix needs more of both. + kubernetesBackupResources: + requests: + cpu: 10m + memory: 64Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 2Gi # The Kafka metadata strategy carries no client image here: unlike the Altinity # and RabbitMQ drivers, which need only curl (plus jq for Altinity), its driver # runs the Kafka Admin-API CLI (kafka-topics / kafka-configs), which only the