mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
feat(backups): add Kubernetes backup strategy for tenant clusters
A tenant Kubernetes application had no backup route in cozy-default, and none of the existing strategies fits it. Its API state is a key prefix in the etcd DataStore the whole tenant shares, so the Etcd strategy's snapshot would roll every tenant cluster back at once; its control-plane CA, service account key and Talos secrets cannot be regenerated without breaking every node and credential issued under them; and its PersistentVolumes are DataVolumes in the tenant namespace that the application's own deletion removes. The new Kubernetes strategy backs up the cluster's etcd prefix with a Job that authenticates with the cluster's own datastore certificate, and the PKI Secrets and data DataVolumes with Velero. The HelmReleases of the cluster and its node pools are recorded on the Backup rather than in Velero, because a label selector for them would also pick up the worker system disks. A backup warns when a CA would expire before the backup does. A restore rebuilds a deleted cluster under its old CA, or rolls a live one back after stopping its control plane and workers, and lets the etcd prefix and the volumes be restored separately. Signed-off-by: Andrey Kolkov <androndo@gmail.com> Assisted-by: LLM
This commit is contained in:
1 parent
5e9c392352
commit
0f44cf9e25
28 files changed
+4697
-22
No files matched your search
@@ -0,0 +1,95 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// Package v1alpha1 defines strategy.backups.cozystack.io API types.
|
||||
//
|
||||
// Group: strategy.backups.cozystack.io
|
||||
// Version: v1alpha1
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
)
|
||||
|
||||
func init() {
|
||||
SchemeBuilder.Register(func(s *runtime.Scheme) error {
|
||||
s.AddKnownTypes(GroupVersion,
|
||||
&Kubernetes{},
|
||||
&KubernetesList{},
|
||||
)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
const (
|
||||
KubernetesStrategyKind = "Kubernetes"
|
||||
)
|
||||
|
||||
// +kubebuilder:object:root=true
|
||||
// +kubebuilder:subresource:status
|
||||
// +kubebuilder:resource:scope=Cluster
|
||||
|
||||
// Kubernetes defines a backup strategy for a tenant Kubernetes application.
|
||||
// A tenant cluster keeps its state in two places the platform can reach: the
|
||||
// cluster's key prefix in the tenant's shared etcd DataStore, and host-side
|
||||
// objects (control-plane PKI Secrets, the application HelmReleases, and the
|
||||
// DataVolumes that back the tenant's PersistentVolumes). The strategy captures
|
||||
// the first with a one-shot Job and the second with Velero. The two are taken
|
||||
// one after another, so a backup is crash-consistent at best.
|
||||
type Kubernetes struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
|
||||
Spec KubernetesSpec `json:"spec,omitempty"`
|
||||
Status KubernetesStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
// +kubebuilder:object:root=true
|
||||
|
||||
// KubernetesList contains a list of Kubernetes backup strategies.
|
||||
type KubernetesList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []Kubernetes `json:"items"`
|
||||
}
|
||||
|
||||
// KubernetesSpec specifies the desired behavior of a Kubernetes backup strategy.
|
||||
type KubernetesSpec struct {
|
||||
// Template holds a PodTemplateSpec for the Job that dumps the cluster's
|
||||
// key prefix from its etcd DataStore to object storage, writes it back on
|
||||
// restore, and deletes the object on cleanup. The Job runs in the
|
||||
// application namespace. Helm-like Go templates are supported in every
|
||||
// string field and are rendered against:
|
||||
// - `.Release.Name` and `.Release.Namespace` — the application's name and
|
||||
// namespace; `.Release.Prefix` is `kubernetes-`.
|
||||
// - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`.
|
||||
// - `.ObjectKey` — the object-storage key of this backup's dump.
|
||||
// - `.Datastore.Prefix` — the cluster's key prefix, without slashes.
|
||||
// - `.Datastore.Endpoints` — comma-separated etcd client URLs.
|
||||
// - `.Datastore.CertificateSecret` — the Secret holding the cluster's
|
||||
// etcd client certificate (`ca.crt`, `server.crt`, `server.key`).
|
||||
// - `.Parameters` — the resolved BackupClass strategy parameters.
|
||||
//
|
||||
// Stored schemaless for the same CRD-size reason as redis_types.go.
|
||||
//
|
||||
// +kubebuilder:validation:Schemaless
|
||||
// +kubebuilder:pruning:PreserveUnknownFields
|
||||
Template corev1.PodTemplateSpec `json:"template"`
|
||||
|
||||
// Velero holds the Velero Backup and Restore specs for the host-side
|
||||
// objects and volumes. Rendered against `.Application` and `.Parameters`,
|
||||
// like the Velero strategy.
|
||||
//
|
||||
// +kubebuilder:validation:Schemaless
|
||||
// +kubebuilder:pruning:PreserveUnknownFields
|
||||
Velero VeleroTemplate `json:"velero"`
|
||||
|
||||
// CertificateWarningThreshold is how close to expiry a control-plane
|
||||
// certificate may be before a backup records a warning. Defaults to 720h.
|
||||
// +optional
|
||||
CertificateWarningThreshold *metav1.Duration `json:"certificateWarningThreshold,omitempty"`
|
||||
}
|
||||
|
||||
type KubernetesStatus struct {
|
||||
Conditions []metav1.Condition `json:"conditions,omitempty"`
|
||||
}
|
||||
+103
@@ -839,6 +839,109 @@ func (in *KafkaStatus) DeepCopy() *KafkaStatus {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Kubernetes) DeepCopyInto(out *Kubernetes) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
|
||||
in.Spec.DeepCopyInto(&out.Spec)
|
||||
in.Status.DeepCopyInto(&out.Status)
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kubernetes.
|
||||
func (in *Kubernetes) DeepCopy() *Kubernetes {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(Kubernetes)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||
func (in *Kubernetes) DeepCopyObject() runtime.Object {
|
||||
if c := in.DeepCopy(); c != nil {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KubernetesList) DeepCopyInto(out *KubernetesList) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
in.ListMeta.DeepCopyInto(&out.ListMeta)
|
||||
if in.Items != nil {
|
||||
in, out := &in.Items, &out.Items
|
||||
*out = make([]Kubernetes, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesList.
|
||||
func (in *KubernetesList) DeepCopy() *KubernetesList {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(KubernetesList)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||
func (in *KubernetesList) DeepCopyObject() runtime.Object {
|
||||
if c := in.DeepCopy(); c != nil {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) {
|
||||
*out = *in
|
||||
in.Template.DeepCopyInto(&out.Template)
|
||||
in.Velero.DeepCopyInto(&out.Velero)
|
||||
if in.CertificateWarningThreshold != nil {
|
||||
in, out := &in.CertificateWarningThreshold, &out.CertificateWarningThreshold
|
||||
*out = new(v1.Duration)
|
||||
**out = **in
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec.
|
||||
func (in *KubernetesSpec) DeepCopy() *KubernetesSpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(KubernetesSpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) {
|
||||
*out = *in
|
||||
if in.Conditions != nil {
|
||||
in, out := &in.Conditions, &out.Conditions
|
||||
*out = make([]v1.Condition, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus.
|
||||
func (in *KubernetesStatus) DeepCopy() *KubernetesStatus {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(KubernetesStatus)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *MariaDB) DeepCopyInto(out *MariaDB) {
|
||||
*out = *in
|
||||
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/bin/bash
|
||||
# Shared helpers for the tenant Kubernetes backup/restore walkthrough.
|
||||
# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh"
|
||||
|
||||
export RED='\033[0;31m'
|
||||
export GREEN='\033[0;32m'
|
||||
export YELLOW='\033[1;33m'
|
||||
export BLUE='\033[0;34m'
|
||||
export MAGENTA='\033[0;35m'
|
||||
export CYAN='\033[0;36m'
|
||||
export NC='\033[0m'
|
||||
export BOLD='\033[1m'
|
||||
|
||||
export NAMESPACE="${NAMESPACE:-tenant-root}"
|
||||
export CLUSTER_NAME="${CLUSTER_NAME:-backup-demo}"
|
||||
export POOL_NAME="${POOL_NAME:-md0}"
|
||||
export WORKER_DISK_SIZE="${WORKER_DISK_SIZE:-20Gi}"
|
||||
export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-cozy-default}"
|
||||
export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kubernetes-backup-job}"
|
||||
export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kubernetes-restore-inplace}"
|
||||
export RESTOREJOB_RECREATE_NAME="${RESTOREJOB_RECREATE_NAME:-kubernetes-restore-recreate}"
|
||||
export TENANT_API_PORT="${TENANT_API_PORT:-16443}"
|
||||
export MARKER_VALUE="${MARKER_VALUE:-roundtrip-7c2e91d4}"
|
||||
export RELEASE="kubernetes-${CLUSTER_NAME}"
|
||||
|
||||
log_info() { echo -e "${BLUE}i${NC} $*" >&2; }
|
||||
log_success() { echo -e "${GREEN}OK${NC} $*" >&2; }
|
||||
log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; }
|
||||
log_error() { echo -e "${RED}x${NC} $*" >&2; }
|
||||
log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; }
|
||||
|
||||
print_header() {
|
||||
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
|
||||
}
|
||||
|
||||
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
|
||||
|
||||
# The admin kubeconfig points at the in-cluster Service, so the walkthrough
|
||||
# reaches the tenant API through a port-forward to it and keeps the TLS server
|
||||
# name the certificate is issued for.
|
||||
TENANT_KUBECONFIG="$(dirname "${BASH_SOURCE[0]}")/.tenant-kubeconfig"
|
||||
export TENANT_KUBECONFIG
|
||||
|
||||
tenant_connect() {
|
||||
kubectl -n "$NAMESPACE" get secret "${RELEASE}-admin-kubeconfig" \
|
||||
-o jsonpath='{.data.super-admin\.svc}' | base64 -d > "$TENANT_KUBECONFIG"
|
||||
chmod 600 "$TENANT_KUBECONFIG"
|
||||
local cluster
|
||||
cluster=$(kubectl --kubeconfig "$TENANT_KUBECONFIG" config view -o jsonpath='{.clusters[0].name}')
|
||||
kubectl --kubeconfig "$TENANT_KUBECONFIG" config set-cluster "$cluster" \
|
||||
--server="https://127.0.0.1:${TENANT_API_PORT}" \
|
||||
--tls-server-name="${RELEASE}.${NAMESPACE}.svc" >/dev/null
|
||||
pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true
|
||||
kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}" "${TENANT_API_PORT}:6443" >/dev/null 2>&1 &
|
||||
for _ in $(seq 1 30); do
|
||||
tkubectl get --raw /readyz >/dev/null 2>&1 && return 0
|
||||
sleep 2
|
||||
done
|
||||
log_error "tenant API of ${RELEASE} is not reachable through the port-forward"
|
||||
return 1
|
||||
}
|
||||
|
||||
tenant_disconnect() {
|
||||
pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true
|
||||
}
|
||||
|
||||
tkubectl() {
|
||||
kubectl --kubeconfig "$TENANT_KUBECONFIG" "$@"
|
||||
}
|
||||
|
||||
# The CA fingerprint is what proves a restore brought back the cluster rather
|
||||
# than a new one: nodes, kubeconfigs and tokens all trust this CA.
|
||||
cluster_ca_fingerprint() {
|
||||
kubectl -n "$NAMESPACE" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' \
|
||||
| base64 -d | openssl x509 -noout -fingerprint -sha256 | cut -d= -f2
|
||||
}
|
||||
|
||||
# wait_tenant_marker checks both halves of the backup: the ConfigMap comes back
|
||||
# with etcd, the file on the PVC with the volumes.
|
||||
wait_tenant_marker() {
|
||||
local timeout="${1:-900}" elapsed=0 cm data
|
||||
while true; do
|
||||
cm=$(tkubectl -n demo get configmap marker -o jsonpath='{.data.marker}' 2>/dev/null || true)
|
||||
data=$(tkubectl -n demo exec deploy/writer -- cat /data/marker 2>/dev/null || true)
|
||||
if [[ "$cm" == "$MARKER_VALUE" && "$data" == "$MARKER_VALUE" ]]; then
|
||||
log_success "marker present in etcd (ConfigMap) and on the volume"
|
||||
return 0
|
||||
fi
|
||||
(( elapsed >= timeout )) && { log_error "marker not restored: configmap='${cm}' volume='${data}'"; return 1; }
|
||||
sleep 10
|
||||
elapsed=$((elapsed + 10))
|
||||
done
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
#!/bin/bash
|
||||
# Step 01: Create a tenant Kubernetes cluster with one worker pool and seed it
|
||||
# with a marker held in etcd (a ConfigMap) and on a PersistentVolume.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/00-helpers.sh"
|
||||
|
||||
print_header "Step 01: Create Kubernetes '${CLUSTER_NAME}' and seed a marker"
|
||||
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: apps.cozystack.io/v1alpha1
|
||||
kind: Kubernetes
|
||||
metadata:
|
||||
name: ${CLUSTER_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
storageClass: replicated
|
||||
---
|
||||
apiVersion: apps.cozystack.io/v1alpha1
|
||||
kind: KubernetesNodes
|
||||
metadata:
|
||||
name: ${CLUSTER_NAME}-${POOL_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
cluster: ${CLUSTER_NAME}
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
instanceType: u1.medium
|
||||
diskSize: ${WORKER_DISK_SIZE}
|
||||
EOF
|
||||
|
||||
wait_hr_ready "$RELEASE" 1200
|
||||
log_substep "Waiting for the worker node to join..."
|
||||
tenant_connect
|
||||
kubectl_ok=0
|
||||
for _ in $(seq 1 120); do
|
||||
if tkubectl wait --for=condition=Ready node --all --timeout=10s >/dev/null 2>&1 \
|
||||
&& [[ -n "$(tkubectl get nodes -o name)" ]] \
|
||||
&& [[ -n "$(tkubectl get storageclass -o name)" ]]; then
|
||||
kubectl_ok=1
|
||||
break
|
||||
fi
|
||||
sleep 15
|
||||
done
|
||||
(( kubectl_ok )) || { log_error "worker node or StorageClass did not appear"; exit 1; }
|
||||
|
||||
tkubectl apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: demo
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: marker
|
||||
namespace: demo
|
||||
data:
|
||||
marker: ${MARKER_VALUE}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: data
|
||||
namespace: demo
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: writer
|
||||
namespace: demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels: {app: writer}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: writer}
|
||||
spec:
|
||||
containers:
|
||||
- name: writer
|
||||
image: busybox:1.36
|
||||
command: ["sh", "-c", "[ -f /data/marker ] || echo -n ${MARKER_VALUE} > /data/marker; sync; sleep infinity"]
|
||||
volumeMounts:
|
||||
- {name: data, mountPath: /data}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim: {claimName: data}
|
||||
EOF
|
||||
|
||||
wait_tenant_marker 900
|
||||
cluster_ca_fingerprint > "$SCRIPT_DIR/.ca-fingerprint"
|
||||
log_success "cluster CA $(cat "$SCRIPT_DIR/.ca-fingerprint")"
|
||||
tenant_disconnect
|
||||
|
||||
echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupjob.sh"
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Step 02: Back the cluster up through the cozy-default BackupClass. The
|
||||
# controller dumps the cluster's etcd prefix to the platform bucket with a
|
||||
# Job, then backs up its control-plane Secrets and volumes with Velero.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/00-helpers.sh"
|
||||
|
||||
print_header "Step 02: Create BackupJob '${BACKUPJOB_NAME}'"
|
||||
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: backups.cozystack.io/v1alpha1
|
||||
kind: BackupJob
|
||||
metadata:
|
||||
name: ${BACKUPJOB_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
applicationRef:
|
||||
apiGroup: apps.cozystack.io
|
||||
kind: Kubernetes
|
||||
name: ${CLUSTER_NAME}
|
||||
backupClassName: ${BACKUPCLASS_NAME}
|
||||
EOF
|
||||
|
||||
wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 1800 Failed
|
||||
|
||||
warn=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="CertificatesExpiring")].status}')
|
||||
if [[ "$warn" == "True" ]]; then
|
||||
log_warning "$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" \
|
||||
-o jsonpath='{.status.conditions[?(@.type=="CertificatesExpiring")].message}')"
|
||||
fi
|
||||
log_success "Backup '$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')' is Ready."
|
||||
|
||||
echo -e "\n${GREEN}${BOLD}Next:${NC} ./03-restore-in-place.sh"
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# Step 03: Lose the demo namespace inside the live cluster and roll the cluster
|
||||
# back. The restore stops the control plane and the workers, writes the etcd
|
||||
# prefix and the volumes back, and returns the cluster to its replica counts.
|
||||
# Set RESTORE_OPTIONS='{"volumes":false}' (or '{"etcd":false}') to restore one
|
||||
# half only.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/00-helpers.sh"
|
||||
|
||||
print_header "Step 03: Restore '${CLUSTER_NAME}' in place"
|
||||
|
||||
tenant_connect
|
||||
log_substep "Deleting namespace demo inside the tenant cluster..."
|
||||
tkubectl delete namespace demo --wait=true --timeout=10m
|
||||
tenant_disconnect
|
||||
|
||||
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')
|
||||
options="${RESTORE_OPTIONS:-}"
|
||||
[[ -n "$options" ]] || options='{}'
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: backups.cozystack.io/v1alpha1
|
||||
kind: RestoreJob
|
||||
metadata:
|
||||
name: ${RESTOREJOB_INPLACE_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
backupRef:
|
||||
name: ${backup_ref}
|
||||
options: ${options}
|
||||
EOF
|
||||
|
||||
wait_for_field restorejob "$RESTOREJOB_INPLACE_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 2400 Failed
|
||||
wait_hr_ready "$RELEASE" 1200
|
||||
|
||||
tenant_connect
|
||||
wait_tenant_marker 1200
|
||||
[[ "$(cluster_ca_fingerprint)" == "$(cat "$SCRIPT_DIR/.ca-fingerprint")" ]] \
|
||||
|| { log_error "cluster CA changed across the restore"; exit 1; }
|
||||
log_success "same cluster CA, marker back in etcd and on the volume"
|
||||
tenant_disconnect
|
||||
|
||||
echo -e "\n${GREEN}${BOLD}Next:${NC} ./04-restore-after-delete.sh"
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
# Step 04: Delete the Kubernetes application and its worker pool, which also
|
||||
# deletes the cluster's etcd prefix and volumes, then rebuild it from the
|
||||
# backup under the same name and CA.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/00-helpers.sh"
|
||||
|
||||
print_header "Step 04: Delete '${CLUSTER_NAME}' and rebuild it from the backup"
|
||||
|
||||
kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found --wait=false
|
||||
kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --wait=false
|
||||
wait_deleted helmrelease "$RELEASE" "$NAMESPACE" 1200
|
||||
wait_deleted tenantcontrolplane "$RELEASE" "$NAMESPACE" 600
|
||||
log_substep "Removing what the application's deletion leaves behind, so the rebuild proves itself..."
|
||||
kubectl -n "$NAMESPACE" delete secret "${RELEASE}-talos-ca" "${RELEASE}-talos-tls-cert" "${RELEASE}-admin-kubeconfig" --ignore-not-found
|
||||
|
||||
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: backups.cozystack.io/v1alpha1
|
||||
kind: RestoreJob
|
||||
metadata:
|
||||
name: ${RESTOREJOB_RECREATE_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
backupRef:
|
||||
name: ${backup_ref}
|
||||
EOF
|
||||
|
||||
wait_for_field restorejob "$RESTOREJOB_RECREATE_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 2400 Failed
|
||||
wait_hr_ready "$RELEASE" 1200
|
||||
|
||||
tenant_connect
|
||||
wait_tenant_marker 1500
|
||||
[[ "$(cluster_ca_fingerprint)" == "$(cat "$SCRIPT_DIR/.ca-fingerprint")" ]] \
|
||||
|| { log_error "cluster CA changed across the rebuild"; exit 1; }
|
||||
log_success "cluster rebuilt with the same CA, marker back in etcd and on the volume"
|
||||
tenant_disconnect
|
||||
@@ -0,0 +1,68 @@
|
||||
# Tenant Kubernetes backup and restore
|
||||
|
||||
The `Kubernetes` strategy backs up a tenant Kubernetes application in two halves, taken one after the other, so a backup is crash-consistent at best:
|
||||
|
||||
- **etcd**: a Job dumps the cluster's key prefix from the tenant's etcd DataStore to the platform bucket. This is the cluster's API state.
|
||||
- **Velero**: the control-plane and Talos PKI Secrets, and the DataVolumes and PVCs that back the cluster's PersistentVolumes. Worker system disks are left out; workers are recreated and join with the restored PKI.
|
||||
|
||||
The Backup also records the HelmReleases of the cluster and of its worker pools. A backup warns, through a `CertificatesExpiring` condition on the BackupJob, when a control-plane CA expires before the backup's TTL runs out or a leaf certificate expires within `certificateWarningThreshold` (720h).
|
||||
|
||||
## Admin: prepare
|
||||
|
||||
Nothing to do when the platform default backups are on: `cozy-default` routes `Kubernetes` to the `cozy-default-kubernetes` strategy.
|
||||
|
||||
## User: back up
|
||||
|
||||
```yaml
|
||||
apiVersion: backups.cozystack.io/v1alpha1
|
||||
kind: BackupJob
|
||||
metadata:
|
||||
name: my-cluster-backup
|
||||
namespace: tenant-foo
|
||||
spec:
|
||||
applicationRef:
|
||||
apiGroup: apps.cozystack.io
|
||||
kind: Kubernetes
|
||||
name: my-cluster
|
||||
backupClassName: cozy-default
|
||||
```
|
||||
|
||||
A `Plan` with the same `applicationRef` schedules it.
|
||||
|
||||
## User: restore
|
||||
|
||||
A restore always goes into the application the backup was taken from.
|
||||
|
||||
```yaml
|
||||
apiVersion: backups.cozystack.io/v1alpha1
|
||||
kind: RestoreJob
|
||||
metadata:
|
||||
name: my-cluster-restore
|
||||
namespace: tenant-foo
|
||||
spec:
|
||||
backupRef:
|
||||
name: my-cluster-backup
|
||||
options:
|
||||
etcd: true # restore the API state (default)
|
||||
volumes: true # restore the PersistentVolume data (default)
|
||||
```
|
||||
|
||||
- **Cluster deleted**: the restore rebuilds it. It restores the PKI Secrets and the volumes, recreates the cluster, writes the etcd state under the new control plane's prefix, and then recreates the worker pools, so the cluster comes back on its old state and its old CA. The cluster and pool HelmReleases are recreated with the spec recorded at backup time, so a backup taken before a platform upgrade that changed the application's values brings back the older values, which the upgraded chart may need migrated by hand.
|
||||
- **Cluster live**: the restore rolls it back. It must be the cluster the backup was taken from: one deleted and created again under the same name has a new CA and service-account key, so the restore refuses it, and the way back to the backed-up cluster is to delete the application and run the restore again as a rebuild. It suspends the cluster's HelmReleases, stops the workers and the cluster-autoscaler when `volumes` are restored and then the control plane when `etcd` is, writes the selected halves back, and returns every replica count it changed. Workers are recreated, so expect the tenant's Pods to restart. Deleting the RestoreJob midway also returns the replica counts, unless its etcd write is under way.
|
||||
|
||||
Nothing is changed until the restore has confirmed that the backup's Velero half and etcd dump exist. If the etcd write fails partway, the control plane stays stopped and the RestoreJob points at the copy of the prefix it saved before writing; run a new RestoreJob to retry.
|
||||
|
||||
Restoring only one half leaves the other as it is now: `etcd` alone keeps the current volume contents, `volumes` alone keeps the current API objects.
|
||||
|
||||
The two halves differ in what a failed restore leaves behind. The etcd half saves the current prefix before it writes and can be retried from that copy. The volume half has no such copy: Velero cannot overwrite an existing claim, so a live-cluster restore deletes the current DataVolumes and PVCs before it restores them, and if the Velero restore then fails, the volume data exists only in the backup.
|
||||
|
||||
The etcd round trip scales with the number of keys under the cluster's prefix. The backup Job holds the whole dump in memory while it checks it, and the restore writes one key per `etcdctl` call: on a test cluster a 441-key restore took about 40 seconds including Pod start. At that rate the default `kubernetesBackupActiveDeadlineSeconds` (1h) covers a prefix of a few tens of thousands of keys. For a cluster with a larger API state, raise it together with the memory limit in `kubernetesBackupResources`.
|
||||
|
||||
## Walkthrough
|
||||
|
||||
```bash
|
||||
./run-all.sh # create, back up, roll back in place, rebuild after delete
|
||||
./cleanup.sh
|
||||
```
|
||||
|
||||
`NAMESPACE`, `CLUSTER_NAME` and `WORKER_DISK_SIZE` are read from the environment; see `00-helpers.sh`.
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
# Remove everything the walkthrough created. Deleting the BackupJob's Backup
|
||||
# removes its Velero backup and its etcd dump object.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/00-helpers.sh"
|
||||
|
||||
print_header "Cleanup"
|
||||
tenant_disconnect
|
||||
kubectl -n "$NAMESPACE" delete restorejob "$RESTOREJOB_INPLACE_NAME" "$RESTOREJOB_RECREATE_NAME" --ignore-not-found
|
||||
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}' 2>/dev/null || true)
|
||||
kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found
|
||||
[[ -n "$backup_ref" ]] && kubectl -n "$NAMESPACE" delete backup "$backup_ref" --ignore-not-found
|
||||
kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found
|
||||
kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --ignore-not-found
|
||||
rm -f "$TENANT_KUBECONFIG" "$SCRIPT_DIR/.ca-fingerprint"
|
||||
log_success "done"
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# End-to-end tenant Kubernetes backup/restore walkthrough. It proves the
|
||||
# restore brings back the same cluster: the CA fingerprint is unchanged, and a
|
||||
# marker kept in etcd and one kept on a PersistentVolume both come back, first
|
||||
# after rolling a live cluster back, then after rebuilding a deleted one.
|
||||
#
|
||||
# Needs the platform default backups (backupstrategy-controller with a Velero
|
||||
# BSL and the system bucket), so cozy-default routes Kubernetes to
|
||||
# cozy-default-kubernetes.
|
||||
#
|
||||
# Env knobs: see 00-helpers.sh. SKIP_RESTORE=1 stops after the backup.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/00-helpers.sh"
|
||||
|
||||
kubectl wait --for=create kubernetes.strategy.backups.cozystack.io/cozy-default-kubernetes --timeout=15m >/dev/null \
|
||||
|| { log_error "cozy-default-kubernetes strategy is missing: enable the platform default backups first"; exit 1; }
|
||||
|
||||
steps=(01 02 03 04)
|
||||
[[ "${SKIP_RESTORE:-0}" == "1" ]] && steps=(01 02)
|
||||
for step in "${steps[@]}"; do
|
||||
script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)"
|
||||
[[ -n "$script" ]] || { echo "no script found for step $step" >&2; exit 1; }
|
||||
echo "=== Running $(basename "$script") ==="
|
||||
bash "$script"
|
||||
done
|
||||
|
||||
print_header "Walkthrough complete"
|
||||
@@ -183,6 +183,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
|
||||
// deletes the object and WAITS for that delete before the Backup is
|
||||
// removed, so no object is orphaned (see cleanupKafkaBackup).
|
||||
return r.cleanupKafkaBackup(ctx, backup)
|
||||
case strategyv1alpha1.KubernetesStrategyKind:
|
||||
// Owns two artifacts: the Velero backup and the etcd dump object.
|
||||
return r.cleanupKubernetesBackup(ctx, backup)
|
||||
case strategyv1alpha1.VeleroStrategyKind:
|
||||
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
||||
default:
|
||||
|
||||
@@ -214,6 +214,8 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
|
||||
return r.reconcileRedis(ctx, j, resolved)
|
||||
case strategyv1alpha1.KafkaStrategyKind:
|
||||
return r.reconcileKafka(ctx, j, resolved)
|
||||
case strategyv1alpha1.KubernetesStrategyKind:
|
||||
return r.reconcileKubernetes(ctx, j, resolved)
|
||||
default:
|
||||
logger.V(1).Info("BackupJob resolved StrategyRef.Kind not supported, skipping",
|
||||
"backupjob", j.Name,
|
||||
@@ -240,6 +242,7 @@ func supportedBackupStrategyKinds() []string {
|
||||
strategyv1alpha1.RabbitmqStrategyKind,
|
||||
strategyv1alpha1.RedisStrategyKind,
|
||||
strategyv1alpha1.KafkaStrategyKind,
|
||||
strategyv1alpha1.KubernetesStrategyKind,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ func TestSupportedBackupStrategyKindsMatchesDispatch(t *testing.T) {
|
||||
strategyv1alpha1.RabbitmqStrategyKind,
|
||||
strategyv1alpha1.RedisStrategyKind,
|
||||
strategyv1alpha1.KafkaStrategyKind,
|
||||
strategyv1alpha1.KubernetesStrategyKind,
|
||||
}
|
||||
sort.Strings(got)
|
||||
sort.Strings(want)
|
||||
|
||||
@@ -593,12 +593,13 @@ func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *ba
|
||||
group = *strategy.StrategyRef.APIGroup
|
||||
}
|
||||
// When the Velero BSL is disabled (velero.bslEnabled=false, surfaced
|
||||
// here as an empty VeleroNamespace) the chart gates the Velero
|
||||
// Strategy CRs off the SAME flag, so they are never rendered. The
|
||||
// BackupClass still routes to them unconditionally, so counting them
|
||||
// as missing would force a Helm upgrade every MinForceInterval
|
||||
// forever against a render that can never produce them.
|
||||
if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" {
|
||||
// here as an empty VeleroNamespace) the chart gates the Strategy CRs
|
||||
// that back up through Velero (Velero, Kubernetes) off the SAME flag,
|
||||
// so they are never rendered. The BackupClass still routes to them
|
||||
// unconditionally, so counting them as missing would force a Helm
|
||||
// upgrade every MinForceInterval forever against a render that can
|
||||
// never produce them.
|
||||
if g.VeleroNamespace == "" && group == strategyAPIGroup && (kind == "Velero" || kind == "Kubernetes") {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%s/%s", kind, name)
|
||||
|
||||
@@ -37,6 +37,7 @@ var gateGVKs = []schema.GroupVersionKind{
|
||||
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"},
|
||||
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"},
|
||||
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"},
|
||||
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Kubernetes"},
|
||||
{Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"},
|
||||
{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"},
|
||||
}
|
||||
@@ -109,6 +110,7 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass {
|
||||
ref("Altinity", "cozy-default-altinity"),
|
||||
ref("Velero", "cozy-default-velero-vminstance"),
|
||||
ref("Velero", "cozy-default-velero-vmdisk"),
|
||||
ref("Kubernetes", "cozy-default-kubernetes"),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -149,6 +151,7 @@ func allDefaultObjects() []runtime.Object {
|
||||
strategyObject("Altinity", "cozy-default-altinity"),
|
||||
strategyObject("Velero", "cozy-default-velero-vminstance"),
|
||||
strategyObject("Velero", "cozy-default-velero-vmdisk"),
|
||||
strategyObject("Kubernetes", "cozy-default-kubernetes"),
|
||||
bslObject(),
|
||||
}
|
||||
}
|
||||
@@ -288,8 +291,8 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) {
|
||||
t.Fatal("expected a forced Helm upgrade when default objects are missing")
|
||||
}
|
||||
// 6 strategyRefs (Velero twice under different names) + the BSL.
|
||||
if len(missing) != 7 {
|
||||
t.Fatalf("missing = %v, want 7 entries", missing)
|
||||
if len(missing) != 8 {
|
||||
t.Fatalf("missing = %v, want 8 entries", missing)
|
||||
}
|
||||
ann := forceAnnotations(t, dyn)
|
||||
forceAt, ok := ann["reconcile.fluxcd.io/forceAt"]
|
||||
@@ -319,6 +322,7 @@ func TestCheckNoopWhenAllPresent(t *testing.T) {
|
||||
strategyObject("Altinity", "cozy-default-altinity"),
|
||||
strategyObject("Velero", "cozy-default-velero-vminstance"),
|
||||
strategyObject("Velero", "cozy-default-velero-vmdisk"),
|
||||
strategyObject("Kubernetes", "cozy-default-kubernetes"),
|
||||
bslObject(),
|
||||
)
|
||||
|
||||
@@ -454,8 +458,8 @@ func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) {
|
||||
if !forced {
|
||||
t.Fatal("objects release throttled by the earlier credentials force")
|
||||
}
|
||||
if len(missing) != 7 {
|
||||
t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing)
|
||||
if len(missing) != 8 {
|
||||
t.Fatalf("missing = %v, want the 7 strategies + the BSL", missing)
|
||||
}
|
||||
if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
|
||||
t.Errorf("own HelmRelease not forced: %v", ann)
|
||||
@@ -482,7 +486,7 @@ func TestForceSkipsSuspendedHelmRelease(t *testing.T) {
|
||||
if forced {
|
||||
t.Error("forced reported true for a suspended HelmRelease that ignores the annotations")
|
||||
}
|
||||
if len(missing) != 7 {
|
||||
if len(missing) != 8 {
|
||||
t.Fatalf("missing = %v, want the objects still reported", missing)
|
||||
}
|
||||
if ann := forceAnnotations(t, dyn); len(ann) != 0 {
|
||||
@@ -599,6 +603,7 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) {
|
||||
strategyObject("Altinity", "cozy-default-altinity"),
|
||||
strategyObject("Velero", "cozy-default-velero-vminstance"),
|
||||
strategyObject("Velero", "cozy-default-velero-vmdisk"),
|
||||
strategyObject("Kubernetes", "cozy-default-kubernetes"),
|
||||
bslObject(),
|
||||
)
|
||||
|
||||
@@ -658,6 +663,7 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) {
|
||||
strategyObject("Altinity", "cozy-default-altinity"),
|
||||
strategyObject("Velero", "cozy-default-velero-vminstance"),
|
||||
strategyObject("Velero", "cozy-default-velero-vmdisk"),
|
||||
strategyObject("Kubernetes", "cozy-default-kubernetes"),
|
||||
)
|
||||
// Rebuild the RESTMapper without BackupStorageLocation: the velero.io API
|
||||
// is not served on this cluster.
|
||||
@@ -1053,8 +1059,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) {
|
||||
g.lastForce = base.Add(-time.Minute)
|
||||
g.now = func() time.Time { return base }
|
||||
|
||||
if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 {
|
||||
t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err)
|
||||
if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != len(allDefaultObjects()) {
|
||||
t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want %d reported, none forced", missing, forced, err, len(allDefaultObjects()))
|
||||
}
|
||||
|
||||
g.now = func() time.Time { return base.Add(5 * time.Minute) }
|
||||
@@ -1065,8 +1071,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) {
|
||||
|
||||
g.now = func() time.Time { return base.Add(time.Hour) }
|
||||
missing, forced, err := g.Check(context.Background())
|
||||
if err != nil || !forced || len(missing) != 7 {
|
||||
t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err)
|
||||
if err != nil || !forced || len(missing) != len(allDefaultObjects()) {
|
||||
t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want %d reported and forced again", missing, forced, err, len(allDefaultObjects()))
|
||||
}
|
||||
if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first {
|
||||
t.Errorf("not forced again for unannotated objects: forceAt still %q", first)
|
||||
|
||||
@@ -0,0 +1,862 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/dynamic"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/template"
|
||||
)
|
||||
|
||||
const (
|
||||
kubernetesAppKind = "Kubernetes"
|
||||
kubernetesReleasePrefix = "kubernetes-"
|
||||
|
||||
kubernetesLabelMode = "kubernetes.strategy.backups.cozystack.io/mode"
|
||||
kubernetesModeBackup = "backup"
|
||||
kubernetesModeRestore = "restore"
|
||||
kubernetesModeCleanup = "cleanup"
|
||||
// kubernetesModeVerify checks that the dump object exists before a restore
|
||||
// changes anything.
|
||||
kubernetesModeVerify = "verify"
|
||||
|
||||
// kubernetesCleanupAttemptsAnnotation counts failed cleanup Jobs on the
|
||||
// Backup, so a persistently failing delete (revoked credentials, a removed
|
||||
// bucket) ends in a released Backup and a Warning rather than a Job storm.
|
||||
kubernetesCleanupAttemptsAnnotation = "kubernetes.strategy.backups.cozystack.io/cleanup-attempts"
|
||||
kubernetesCleanupMaxAttempts = 5
|
||||
|
||||
kubernetesMetaPrefix = "kubernetes.strategy.backups.cozystack.io/"
|
||||
kubernetesObjectMetaKey = kubernetesMetaPrefix + "object"
|
||||
kubernetesDatastorePrefixMetaKey = kubernetesMetaPrefix + "datastore-prefix"
|
||||
kubernetesDatastoreEndpointsMetaKey = kubernetesMetaPrefix + "datastore-endpoints"
|
||||
kubernetesHelmReleasesMetaKey = kubernetesMetaPrefix + "helmreleases"
|
||||
kubernetesDataVolumesMetaKey = kubernetesMetaPrefix + "datavolumes"
|
||||
kubernetesCertWarningsMetaKey = kubernetesMetaPrefix + "certificate-warnings"
|
||||
kubernetesControlPlaneUIDMetaKey = kubernetesMetaPrefix + "tenantcontrolplane-uid"
|
||||
kubernetesParamPrefix = kubernetesMetaPrefix + "parameter/"
|
||||
|
||||
// kubernetesCondCertificates is set on the BackupJob once the control-plane
|
||||
// certificates have been checked; Status=True means at least one expires
|
||||
// inside the warning window.
|
||||
kubernetesCondCertificates = "CertificatesExpiring"
|
||||
|
||||
kubernetesPollInterval = 5 * time.Second
|
||||
// Leaf certificates are rotated by Kamaji and cert-manager, so a near
|
||||
// expiry is only worth a warning when it is this close.
|
||||
kubernetesDefaultCertThreshold = 720 * time.Hour
|
||||
// A CA must outlive every backup that may still be restored, so its
|
||||
// window is the Velero backup TTL; this is the fallback when none is set.
|
||||
kubernetesDefaultBackupTTL = 720 * time.Hour
|
||||
|
||||
helmReleaseNameAnnotation = "meta.helm.sh/release-name"
|
||||
clusterNameLabel = "cluster.x-k8s.io/cluster-name"
|
||||
helmInstanceLabel = "app.kubernetes.io/instance"
|
||||
kamajiNameLabel = "kamaji.clastix.io/name"
|
||||
)
|
||||
|
||||
var (
|
||||
tenantControlPlaneGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "tenantcontrolplanes"}
|
||||
kamajiDataStoreGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "datastores"}
|
||||
kamajiControlPlaneGVR = schema.GroupVersionResource{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Resource: "kamajicontrolplanes"}
|
||||
machineDeploymentGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"}
|
||||
machineGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machines"}
|
||||
deploymentGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
|
||||
)
|
||||
|
||||
// kubernetesCertificateSecrets lists, by suffix after the release name, the
|
||||
// Secrets whose certificates a backup checks. Kamaji and cert-manager rotate
|
||||
// the leaves; the CAs are irreplaceable, so a restore needs them valid.
|
||||
var kubernetesCertificateSecrets = []string{
|
||||
"ca",
|
||||
"front-proxy-ca-certificate",
|
||||
"api-server-certificate",
|
||||
"api-server-kubelet-client-certificate",
|
||||
"front-proxy-client-certificate",
|
||||
"talos-ca",
|
||||
"talos-tls-cert",
|
||||
}
|
||||
|
||||
// kubernetesUnlabelledSecrets are issued by cert-manager without the Helm
|
||||
// instance label the Velero selector relies on; the driver adds it before the
|
||||
// backup so the Talos PKI is not silently left out.
|
||||
var kubernetesUnlabelledSecrets = []string{"talos-ca", "talos-tls-cert"}
|
||||
|
||||
func kubernetesReleaseName(appName string) string {
|
||||
return kubernetesReleasePrefix + appName
|
||||
}
|
||||
|
||||
func validateKubernetesApplicationRef(ref corev1.TypedLocalObjectReference) error {
|
||||
if ref.Kind != kubernetesAppKind {
|
||||
return fmt.Errorf("kubernetes strategy supports applicationRef.kind %s, got %q", kubernetesAppKind, ref.Kind)
|
||||
}
|
||||
if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup {
|
||||
return fmt.Errorf("kubernetes strategy supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func kubernetesObjectKey(namespace, appName, backupName string) string {
|
||||
return fmt.Sprintf("%s/%s/%s.etcd.json", namespace, appName, backupName)
|
||||
}
|
||||
|
||||
// kubernetesDatastore is where the tenant cluster keeps its state: a key
|
||||
// prefix in the etcd DataStore shared by every cluster of the tenant.
|
||||
type kubernetesDatastore struct {
|
||||
// ControlPlaneUID identifies the cluster across a delete and re-create
|
||||
// under the same name, which Prefix does not on a Kamaji that keys the
|
||||
// schema on the namespace and name.
|
||||
ControlPlaneUID string
|
||||
Prefix string
|
||||
Endpoints []string
|
||||
CertificateSecret string
|
||||
// SourcePrefix is the prefix the restored dump was taken under. It
|
||||
// differs from Prefix when a deleted cluster is rebuilt: Kamaji keys a new
|
||||
// TenantControlPlane's schema on its UID.
|
||||
SourcePrefix string
|
||||
}
|
||||
|
||||
func (d kubernetesDatastore) templateContext() map[string]any {
|
||||
source := d.SourcePrefix
|
||||
if source == "" {
|
||||
source = d.Prefix
|
||||
}
|
||||
return map[string]any{
|
||||
"Prefix": d.Prefix,
|
||||
"SourcePrefix": source,
|
||||
"Endpoints": strings.Join(d.Endpoints, ","),
|
||||
"CertificateSecret": d.CertificateSecret,
|
||||
}
|
||||
}
|
||||
|
||||
// etcdEndpointURL gives a scheme to a DataStore endpoint: Kamaji lists them as
|
||||
// host:port and always talks TLS to an etcd DataStore.
|
||||
func etcdEndpointURL(endpoint string) string {
|
||||
if strings.Contains(endpoint, "://") {
|
||||
return endpoint
|
||||
}
|
||||
return "https://" + endpoint
|
||||
}
|
||||
|
||||
// resolveKubernetesDatastore reads the cluster's key prefix and client
|
||||
// certificate from its TenantControlPlane and the client URLs from the
|
||||
// DataStore it points at.
|
||||
func resolveKubernetesDatastore(ctx context.Context, dyn dynamic.Interface, namespace, appName string) (*kubernetesDatastore, error) {
|
||||
release := kubernetesReleaseName(appName)
|
||||
tcp, err := dyn.Resource(tenantControlPlaneGVR).Namespace(namespace).Get(ctx, release, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if driver, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "driver"); driver != "" && driver != "etcd" {
|
||||
return nil, fmt.Errorf("TenantControlPlane %s/%s uses the %q datastore driver; only etcd is supported", namespace, release, driver)
|
||||
}
|
||||
prefix, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema")
|
||||
if prefix == "" {
|
||||
prefix, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStoreSchema")
|
||||
}
|
||||
if prefix == "" {
|
||||
return nil, fmt.Errorf("TenantControlPlane %s/%s reports no datastore schema yet", namespace, release)
|
||||
}
|
||||
dsName, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "dataStoreName")
|
||||
if dsName == "" {
|
||||
dsName, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStore")
|
||||
}
|
||||
if dsName == "" {
|
||||
return nil, fmt.Errorf("TenantControlPlane %s/%s names no DataStore", namespace, release)
|
||||
}
|
||||
certSecret, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "certificate", "secretName")
|
||||
if certSecret == "" {
|
||||
certSecret = release + "-datastore-certificate"
|
||||
}
|
||||
ds, err := dyn.Resource(kamajiDataStoreGVR).Get(ctx, dsName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get DataStore %s: %w", dsName, err)
|
||||
}
|
||||
raw, _, _ := unstructured.NestedStringSlice(ds.Object, "spec", "endpoints")
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("DataStore %s lists no endpoints", dsName)
|
||||
}
|
||||
endpoints := make([]string, 0, len(raw))
|
||||
for _, e := range raw {
|
||||
endpoints = append(endpoints, etcdEndpointURL(e))
|
||||
}
|
||||
return &kubernetesDatastore{ControlPlaneUID: string(tcp.GetUID()), Prefix: prefix, Endpoints: endpoints, CertificateSecret: certSecret}, nil
|
||||
}
|
||||
|
||||
// kubernetesCertificateWarnings reports every certificate in the given Secrets
|
||||
// that expires inside its window: caWindow for CAs, leafWindow for the rest.
|
||||
// Output is sorted so a condition message is stable across reconciles.
|
||||
func kubernetesCertificateWarnings(secrets []corev1.Secret, now time.Time, caWindow, leafWindow time.Duration) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, s := range secrets {
|
||||
for key, data := range s.Data {
|
||||
if !strings.HasSuffix(key, ".crt") {
|
||||
continue
|
||||
}
|
||||
for rest := data; ; {
|
||||
var block *pem.Block
|
||||
block, rest = pem.Decode(rest)
|
||||
if block == nil {
|
||||
break
|
||||
}
|
||||
if block.Type != "CERTIFICATE" {
|
||||
continue
|
||||
}
|
||||
cert, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
window := leafWindow
|
||||
kind := "certificate"
|
||||
if cert.IsCA {
|
||||
window = caWindow
|
||||
kind = "CA"
|
||||
}
|
||||
if !cert.NotAfter.Before(now.Add(window)) {
|
||||
continue
|
||||
}
|
||||
msg := fmt.Sprintf("%s %s/%s (CN=%s) expires %s", kind, s.Name, key, cert.Subject.CommonName, cert.NotAfter.UTC().Format(time.RFC3339))
|
||||
if !seen[msg] {
|
||||
seen[msg] = true
|
||||
out = append(out, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// kubernetesHelmReleaseNames returns the HelmRelease of the cluster followed
|
||||
// by those of its worker pools. Pools live in separate kubernetes-nodes
|
||||
// releases; the MachineDeployments of the cluster name them, which also
|
||||
// covers the older layout where the cluster release owned its pools.
|
||||
func kubernetesHelmReleaseNames(mds []unstructured.Unstructured, release string) []string {
|
||||
names := []string{release}
|
||||
for _, md := range mds {
|
||||
if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster != release {
|
||||
continue
|
||||
}
|
||||
if owner := md.GetAnnotations()[helmReleaseNameAnnotation]; owner != "" && !slices.Contains(names, owner) {
|
||||
names = append(names, owner)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// sanitizeHelmRelease keeps what is needed to recreate a HelmRelease and
|
||||
// drops server-set metadata and status.
|
||||
func sanitizeHelmRelease(hr *unstructured.Unstructured) map[string]any {
|
||||
labels := map[string]string{}
|
||||
for k, v := range hr.GetLabels() {
|
||||
if !strings.HasPrefix(k, "velero.io/") {
|
||||
labels[k] = v
|
||||
}
|
||||
}
|
||||
meta := map[string]any{
|
||||
"name": hr.GetName(),
|
||||
"namespace": hr.GetNamespace(),
|
||||
}
|
||||
if len(labels) > 0 {
|
||||
meta["labels"] = labels
|
||||
}
|
||||
spec, _, _ := unstructured.NestedMap(hr.Object, "spec")
|
||||
delete(spec, "suspend")
|
||||
return map[string]any{
|
||||
"apiVersion": hr.GetAPIVersion(),
|
||||
"kind": hr.GetKind(),
|
||||
"metadata": meta,
|
||||
"spec": spec,
|
||||
}
|
||||
}
|
||||
|
||||
// kubernetesSchemaPattern is every datastore schema Kamaji assigns: a
|
||||
// TenantControlPlane UID, or <namespace>_<name> on an older Kamaji. The Job
|
||||
// turns a schema into the prefix it runs `etcdctl del --prefix` on in the
|
||||
// etcd every cluster of the tenant shares, so a value that is empty, holds a
|
||||
// slash or a dot, or comes from an edited Backup must never reach it.
|
||||
var kubernetesSchemaPattern = regexp.MustCompile(`^[a-z0-9_-]+$`)
|
||||
|
||||
func validateKubernetesDatastore(ds kubernetesDatastore) error {
|
||||
if !kubernetesSchemaPattern.MatchString(ds.Prefix) {
|
||||
return fmt.Errorf("refusing datastore schema %q: want %s", ds.Prefix, kubernetesSchemaPattern)
|
||||
}
|
||||
if ds.SourcePrefix != "" && !kubernetesSchemaPattern.MatchString(ds.SourcePrefix) {
|
||||
return fmt.Errorf("refusing source datastore schema %q: want %s", ds.SourcePrefix, kubernetesSchemaPattern)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func renderKubernetesTemplate(
|
||||
tmpl corev1.PodTemplateSpec,
|
||||
appName, namespace, mode, objectKey string,
|
||||
ds kubernetesDatastore,
|
||||
parameters map[string]string,
|
||||
) (*corev1.PodTemplateSpec, error) {
|
||||
if err := validateKubernetesDatastore(ds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return template.Template(&tmpl, map[string]any{
|
||||
"Release": map[string]string{
|
||||
"Name": appName,
|
||||
"Namespace": namespace,
|
||||
"Prefix": kubernetesReleasePrefix,
|
||||
},
|
||||
"Mode": mode,
|
||||
"ObjectKey": objectKey,
|
||||
"Datastore": ds.templateContext(),
|
||||
"Parameters": parameters,
|
||||
})
|
||||
}
|
||||
|
||||
func kubernetesStrategyParameters(b *backupsv1alpha1.Backup) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, v := range b.Spec.DriverMetadata {
|
||||
if p := strings.TrimPrefix(k, kubernetesParamPrefix); p != k && p != "" {
|
||||
out[p] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func kubernetesBackupDatastore(b *backupsv1alpha1.Backup) kubernetesDatastore {
|
||||
ds := kubernetesDatastore{
|
||||
Prefix: b.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey],
|
||||
CertificateSecret: kubernetesReleaseName(b.Spec.ApplicationRef.Name) + "-datastore-certificate",
|
||||
}
|
||||
if e := b.Spec.DriverMetadata[kubernetesDatastoreEndpointsMetaKey]; e != "" {
|
||||
ds.Endpoints = strings.Split(e, ",")
|
||||
}
|
||||
return ds
|
||||
}
|
||||
|
||||
func kubernetesBackupTTL(s *strategyv1alpha1.Kubernetes) time.Duration {
|
||||
if s.Spec.Velero.Spec.TTL.Duration > 0 {
|
||||
return s.Spec.Velero.Spec.TTL.Duration
|
||||
}
|
||||
return kubernetesDefaultBackupTTL
|
||||
}
|
||||
|
||||
func kubernetesCertThreshold(s *strategyv1alpha1.Kubernetes) time.Duration {
|
||||
if s.Spec.CertificateWarningThreshold != nil && s.Spec.CertificateWarningThreshold.Duration > 0 {
|
||||
return s.Spec.CertificateWarningThreshold.Duration
|
||||
}
|
||||
return kubernetesDefaultCertThreshold
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BackupJob path
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// reconcileKubernetes drives a BackupJob for a tenant Kubernetes application in
|
||||
// three steps, each observed on later reconciles: check the control-plane
|
||||
// certificates, dump the cluster's etcd prefix with a Job, then back up the
|
||||
// host-side Secrets and volumes with Velero. The Backup records what a restore
|
||||
// needs to put both halves back.
|
||||
func (r *BackupJobReconciler) reconcileKubernetes(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) {
|
||||
logger := getLogger(ctx)
|
||||
logger.Debug("reconciling Kubernetes strategy", "backupjob", j.Name, "phase", j.Status.Phase)
|
||||
|
||||
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded ||
|
||||
j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
if err := validateKubernetesApplicationRef(j.Spec.ApplicationRef); err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, err.Error())
|
||||
}
|
||||
|
||||
if j.Status.StartedAt == nil {
|
||||
fresh := &backupsv1alpha1.BackupJob{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if fresh.Status.StartedAt == nil {
|
||||
base := fresh.DeepCopy()
|
||||
now := metav1.Now()
|
||||
fresh.Status.StartedAt = &now
|
||||
fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
|
||||
if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
j.Status.StartedAt = fresh.Status.StartedAt
|
||||
j.Status.Phase = fresh.Status.Phase
|
||||
}
|
||||
|
||||
strategy := &strategyv1alpha1.Kubernetes{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name)
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef)
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application not found: %s/%s", j.Namespace, j.Spec.ApplicationRef.Name))
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
appName := j.Spec.ApplicationRef.Name
|
||||
release := kubernetesReleaseName(appName)
|
||||
ds, err := resolveKubernetesDatastore(ctx, r.Interface, j.Namespace, appName)
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s has no TenantControlPlane: %v", j.Namespace, appName, err))
|
||||
}
|
||||
return r.markBackupJobFailed(ctx, j, err.Error())
|
||||
}
|
||||
|
||||
// A restore that failed partway leaves the control plane stopped on a
|
||||
// possibly partial prefix, marked with the replica count it had. Backing
|
||||
// that up would turn the damage into a recovery point, so a run that has
|
||||
// not dumped anything yet refuses.
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: jobNameForBackupJob(j)}, &batchv1.Job{}); apierrors.IsNotFound(err) {
|
||||
kcp, kerr := r.Interface.Resource(kamajiControlPlaneGVR).Namespace(j.Namespace).Get(ctx, release, metav1.GetOptions{})
|
||||
if kerr != nil && !apierrors.IsNotFound(kerr) && !apimeta.IsNoMatchError(kerr) {
|
||||
return ctrl.Result{}, kerr
|
||||
}
|
||||
if kerr == nil {
|
||||
if _, stopped := kcp.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; stopped {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s is stopped by an unfinished restore; finish or repair the restore before backing it up", j.Namespace, appName))
|
||||
}
|
||||
}
|
||||
} else if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
if apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates) == nil {
|
||||
return r.checkKubernetesCertificates(ctx, j, strategy, release)
|
||||
}
|
||||
|
||||
objectKey := kubernetesObjectKey(j.Namespace, appName, j.Name)
|
||||
rendered, err := renderKubernetesTemplate(strategy.Spec.Template, appName, j.Namespace, kubernetesModeBackup, objectKey, *ds, resolved.Parameters)
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Kubernetes strategy: %v", err))
|
||||
}
|
||||
dump, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, jobNameForBackupJob(j),
|
||||
map[string]string{
|
||||
kubernetesLabelMode: kubernetesModeBackup,
|
||||
backupsv1alpha1.OwningJobNameLabel: j.Name,
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
|
||||
}, rendered)
|
||||
if err != nil {
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure etcd dump Job: %v", err))
|
||||
}
|
||||
switch jobConditionState(dump) {
|
||||
case batchv1.JobFailed:
|
||||
msg := jobFailureMessage(dump)
|
||||
if msg == "" {
|
||||
msg = "etcd dump Job reported Failed"
|
||||
}
|
||||
return r.markBackupJobFailed(ctx, j, msg)
|
||||
case batchv1.JobComplete:
|
||||
default:
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
|
||||
veleroBackups := &velerov1.BackupList{}
|
||||
if err := r.List(ctx, veleroBackups, client.InNamespace(veleroNamespace), client.MatchingLabels{
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
|
||||
backupsv1alpha1.OwningJobNameLabel: j.Name,
|
||||
}); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if len(veleroBackups.Items) == 0 {
|
||||
if err := r.labelKubernetesSecrets(ctx, j.Namespace, release); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if err := r.createKubernetesVeleroBackup(ctx, j, strategy, app, resolved); err != nil {
|
||||
return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, fmt.Sprintf("failed to create Velero Backup: %v", err))
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
if len(veleroBackups.Items) > 1 {
|
||||
return r.markBackupJobFailed(ctx, j, "more than one Velero Backup references this BackupJob")
|
||||
}
|
||||
vb := &veleroBackups.Items[0]
|
||||
switch vb.Status.Phase {
|
||||
case velerov1.BackupPhaseCompleted:
|
||||
case velerov1.BackupPhaseFailed, velerov1.BackupPhasePartiallyFailed, velerov1.BackupPhaseFailedValidation:
|
||||
return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, vb))
|
||||
default:
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
|
||||
if j.Status.BackupRef != nil {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
// A failed create is retried rather than failing the run: the dump and
|
||||
// the Velero backup already exist, and no Backup would own them.
|
||||
artifact, err := r.createKubernetesBackupArtifact(ctx, j, resolved, vb, *ds, objectKey)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("create Backup artifact: %w", err)
|
||||
}
|
||||
now := metav1.Now()
|
||||
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
|
||||
j.Status.CompletedAt = &now
|
||||
j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: "BackupCompleted",
|
||||
Message: "etcd prefix and Velero backup completed",
|
||||
})
|
||||
return ctrl.Result{}, r.Status().Update(ctx, j)
|
||||
}
|
||||
|
||||
// failKubernetesBackupAfterDump fails a run whose etcd dump is already in the
|
||||
// bucket. No Backup will own the dump, so a cleanup-mode Job owned by the
|
||||
// BackupJob removes it; the dump holds every Secret of the cluster.
|
||||
func (r *BackupJobReconciler) failKubernetesBackupAfterDump(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, ds kubernetesDatastore, objectKey string, resolved *ResolvedBackupConfig, msg string) (ctrl.Result, error) {
|
||||
rendered, err := renderKubernetesTemplate(strategy.Spec.Template, j.Spec.ApplicationRef.Name, j.Namespace, kubernetesModeCleanup, objectKey, ds, resolved.Parameters)
|
||||
if err == nil {
|
||||
_, err = ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, j.Name+"-discard",
|
||||
map[string]string{
|
||||
kubernetesLabelMode: kubernetesModeCleanup,
|
||||
backupsv1alpha1.OwningJobNameLabel: j.Name,
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
|
||||
}, rendered)
|
||||
}
|
||||
if err != nil {
|
||||
msg = fmt.Sprintf("%s; the etcd dump %s could not be scheduled for deletion: %v", msg, objectKey, err)
|
||||
}
|
||||
return r.markBackupJobFailed(ctx, j, msg)
|
||||
}
|
||||
|
||||
// checkKubernetesCertificates records on the BackupJob whether any
|
||||
// control-plane certificate expires inside its window. It never fails the
|
||||
// run: an expiring certificate is a reason to act, not to skip the backup.
|
||||
func (r *BackupJobReconciler) checkKubernetesCertificates(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, release string) (ctrl.Result, error) {
|
||||
var secrets []corev1.Secret
|
||||
for _, suffix := range kubernetesCertificateSecrets {
|
||||
s := corev1.Secret{}
|
||||
err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: release + "-" + suffix}, &s)
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
secrets = append(secrets, s)
|
||||
}
|
||||
warnings := kubernetesCertificateWarnings(secrets, time.Now(), kubernetesBackupTTL(strategy), kubernetesCertThreshold(strategy))
|
||||
cond := metav1.Condition{
|
||||
Type: kubernetesCondCertificates,
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "CertificatesValid",
|
||||
Message: "no control-plane certificate expires within the warning window",
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
cond.Status = metav1.ConditionTrue
|
||||
cond.Reason = "CertificatesExpiring"
|
||||
cond.Message = strings.Join(warnings, "; ")
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Event(j, corev1.EventTypeWarning, "CertificatesExpiring", cond.Message)
|
||||
}
|
||||
}
|
||||
apimeta.SetStatusCondition(&j.Status.Conditions, cond)
|
||||
if err := r.Status().Update(ctx, j); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) labelKubernetesSecrets(ctx context.Context, namespace, release string) error {
|
||||
for _, suffix := range kubernetesUnlabelledSecrets {
|
||||
s := &corev1.Secret{}
|
||||
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release + "-" + suffix}, s)
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Labels[helmInstanceLabel] == release {
|
||||
continue
|
||||
}
|
||||
base := s.DeepCopy()
|
||||
if s.Labels == nil {
|
||||
s.Labels = map[string]string{}
|
||||
}
|
||||
s.Labels[helmInstanceLabel] = release
|
||||
if err := r.Patch(ctx, s, client.MergeFrom(base)); err != nil {
|
||||
return fmt.Errorf("label Secret %s/%s: %w", namespace, s.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) createKubernetesVeleroBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, app map[string]any, resolved *ResolvedBackupConfig) error {
|
||||
spec, err := template.Template(&strategy.Spec.Velero.Spec, map[string]any{
|
||||
"Application": app,
|
||||
"Parameters": resolved.Parameters,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The volumes an in-place restore deletes and replaces are listed before
|
||||
// Velero starts: each of them is then in the Velero backup unless it was
|
||||
// deleted first, and a volume created while the backup runs, which Velero
|
||||
// may have missed, is never deleted by a restore of it.
|
||||
volumes, err := r.kubernetesDataVolumeNames(ctx, j.Namespace, kubernetesReleaseName(j.Spec.ApplicationRef.Name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vb := &velerov1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("%s.%s-", j.Namespace, j.Name),
|
||||
Namespace: veleroNamespace,
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: j.Name,
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
|
||||
},
|
||||
Annotations: map[string]string{kubernetesDataVolumesMetaKey: strings.Join(volumes, ",")},
|
||||
},
|
||||
Spec: *spec,
|
||||
}
|
||||
if err := r.Create(ctx, vb); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Event(j, corev1.EventTypeNormal, "VeleroBackupCreated", fmt.Sprintf("Created Velero Backup %s/%s", veleroNamespace, vb.Name))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// kubernetesDataVolumeNames lists the claims backing the cluster's
|
||||
// PersistentVolumes: the DataVolumes and PVCs the CSI driver labels with the
|
||||
// cluster name. A PVC is listed even without a DataVolume, so an in-place
|
||||
// restore clears it too rather than leaving Velero to skip it.
|
||||
func (r *BackupJobReconciler) kubernetesDataVolumeNames(ctx context.Context, ns, release string) ([]string, error) {
|
||||
selector := clusterNameLabel + "=" + release
|
||||
seen := map[string]bool{}
|
||||
dvs, err := r.Interface.Resource(dataVolumeGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: selector})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, dv := range dvs.Items {
|
||||
seen[dv.GetName()] = true
|
||||
}
|
||||
pvcs := &corev1.PersistentVolumeClaimList{}
|
||||
if err := r.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{clusterNameLabel: release}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, pvc := range pvcs.Items {
|
||||
seen[pvc.Name] = true
|
||||
}
|
||||
names := make([]string, 0, len(seen))
|
||||
for name := range seen {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// createKubernetesBackupArtifact records, besides the Velero backup and the
|
||||
// etcd object, the cluster's HelmReleases and data DataVolumes. The
|
||||
// HelmReleases are kept here rather than in Velero because selecting them by
|
||||
// application label would also select the worker system disks, which the
|
||||
// lineage webhook labels after the same application.
|
||||
func (r *BackupJobReconciler) createKubernetesBackupArtifact(
|
||||
ctx context.Context,
|
||||
j *backupsv1alpha1.BackupJob,
|
||||
resolved *ResolvedBackupConfig,
|
||||
vb *velerov1.Backup,
|
||||
ds kubernetesDatastore,
|
||||
objectKey string,
|
||||
) (*backupsv1alpha1.Backup, error) {
|
||||
release := kubernetesReleaseName(j.Spec.ApplicationRef.Name)
|
||||
mds, err := r.Interface.Resource(machineDeploymentGVR).Namespace(j.Namespace).List(ctx, metav1.ListOptions{})
|
||||
if err != nil && !apierrors.IsNotFound(err) && !apimeta.IsNoMatchError(err) {
|
||||
return nil, err
|
||||
}
|
||||
var mdItems []unstructured.Unstructured
|
||||
if mds != nil {
|
||||
mdItems = mds.Items
|
||||
}
|
||||
var releases []map[string]any
|
||||
for _, name := range kubernetesHelmReleaseNames(mdItems, release) {
|
||||
hr, err := r.Interface.Resource(helmReleaseGVR).Namespace(j.Namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
releases = append(releases, sanitizeHelmRelease(hr))
|
||||
}
|
||||
releasesJSON, err := json.Marshal(releases)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
driverMD := map[string]string{
|
||||
veleroBackupNameMetadataKey: vb.Name,
|
||||
veleroBackupNamespaceMetadataKey: vb.Namespace,
|
||||
kubernetesObjectMetaKey: objectKey,
|
||||
kubernetesDatastorePrefixMetaKey: ds.Prefix,
|
||||
kubernetesDatastoreEndpointsMetaKey: strings.Join(ds.Endpoints, ","),
|
||||
kubernetesHelmReleasesMetaKey: string(releasesJSON),
|
||||
kubernetesDataVolumesMetaKey: vb.Annotations[kubernetesDataVolumesMetaKey],
|
||||
kubernetesControlPlaneUIDMetaKey: ds.ControlPlaneUID,
|
||||
}
|
||||
if c := apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates); c != nil && c.Status == metav1.ConditionTrue {
|
||||
driverMD[kubernetesCertWarningsMetaKey] = c.Message
|
||||
}
|
||||
for k, v := range resolved.Parameters {
|
||||
driverMD[kubernetesParamPrefix+k] = v
|
||||
}
|
||||
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: j.Name, Namespace: j.Namespace},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: j.Spec.ApplicationRef,
|
||||
StrategyRef: resolved.StrategyRef,
|
||||
TakenAt: metav1.Now(),
|
||||
DriverMetadata: driverMD,
|
||||
PlanRef: j.Spec.PlanRef,
|
||||
},
|
||||
Status: backupsv1alpha1.BackupStatus{
|
||||
Phase: backupsv1alpha1.BackupPhaseReady,
|
||||
Artifact: &backupsv1alpha1.BackupArtifact{URI: fmt.Sprintf("velero://%s/%s", vb.Namespace, vb.Name)},
|
||||
},
|
||||
}
|
||||
if err := r.Create(ctx, backup); err != nil {
|
||||
if !apierrors.IsAlreadyExists(err) {
|
||||
return nil, err
|
||||
}
|
||||
existing := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return existing, nil
|
||||
}
|
||||
return backup, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backup deletion cleanup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// cleanupKubernetesBackup removes both halves of a deleted Backup: the Velero
|
||||
// backup through a DeleteBackupRequest, and the etcd dump object through a
|
||||
// cleanup-mode Job, waiting for the Job before the Backup is released. The
|
||||
// escape hatches match the Redis driver's.
|
||||
func (r *BackupReconciler) cleanupKubernetesBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
||||
if err := r.cleanupVeleroBackup(ctx, backup); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
jobName := backup.Name + "-cleanup"
|
||||
if backup.Annotations[redisSkipArtifactCleanupAnnotation] == "true" {
|
||||
existing := &batchv1.Job{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, existing); err == nil {
|
||||
_ = r.deleteRedisCleanupJob(ctx, existing)
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
objectKey := backup.Spec.DriverMetadata[kubernetesObjectMetaKey]
|
||||
if objectKey == "" {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
strategy := &strategyv1alpha1.Kubernetes{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.releaseRedisCleanup(ctx, backup, objectKey, "strategy CR is gone"), nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
job := &batchv1.Job{}
|
||||
err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job)
|
||||
switch {
|
||||
case apierrors.IsNotFound(err):
|
||||
terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace)
|
||||
if nsErr != nil {
|
||||
return ctrl.Result{}, nsErr
|
||||
}
|
||||
if terminating {
|
||||
return r.releaseRedisCleanup(ctx, backup, objectKey, "namespace is terminating"), nil
|
||||
}
|
||||
if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil {
|
||||
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot project credentials: %v", perr)), nil
|
||||
}
|
||||
rendered, rerr := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, backup.Namespace,
|
||||
kubernetesModeCleanup, objectKey, kubernetesBackupDatastore(backup), kubernetesStrategyParameters(backup))
|
||||
if rerr != nil {
|
||||
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cleanup template render failed: %v", rerr)), nil
|
||||
}
|
||||
cleanup := buildRedisCleanupJob(backup.Namespace, jobName, rendered)
|
||||
cleanup.Labels = map[string]string{kubernetesLabelMode: kubernetesModeCleanup}
|
||||
delete(cleanup.Spec.Template.Labels, redisLabelMode)
|
||||
maps.Copy(cleanup.Spec.Template.Labels, cleanup.Labels)
|
||||
if cerr := r.Create(ctx, cleanup); cerr != nil && !apierrors.IsAlreadyExists(cerr) {
|
||||
if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) {
|
||||
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil
|
||||
}
|
||||
return ctrl.Result{}, cerr
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
case err != nil:
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
if !job.DeletionTimestamp.IsZero() {
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
switch jobConditionState(job) {
|
||||
case batchv1.JobComplete:
|
||||
_ = r.deleteRedisCleanupJob(ctx, job)
|
||||
return ctrl.Result{}, nil
|
||||
case batchv1.JobFailed:
|
||||
attempts, _ := strconv.Atoi(backup.Annotations[kubernetesCleanupAttemptsAnnotation])
|
||||
attempts++
|
||||
if attempts >= kubernetesCleanupMaxAttempts {
|
||||
_ = r.deleteRedisCleanupJob(ctx, job)
|
||||
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("the cleanup Job failed %d times: %s", attempts, jobFailureMessage(job))), nil
|
||||
}
|
||||
base := backup.DeepCopy()
|
||||
if backup.Annotations == nil {
|
||||
backup.Annotations = map[string]string{}
|
||||
}
|
||||
backup.Annotations[kubernetesCleanupAttemptsAnnotation] = strconv.Itoa(attempts)
|
||||
if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
_ = r.deleteRedisCleanupJob(ctx, job)
|
||||
return ctrl.Result{RequeueAfter: time.Duration(attempts) * 30 * time.Second}, nil
|
||||
default:
|
||||
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -204,6 +204,8 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request)
|
||||
return r.reconcileRedisRestore(ctx, restoreJob, backup)
|
||||
case strategyv1alpha1.KafkaStrategyKind:
|
||||
return r.reconcileKafkaRestore(ctx, restoreJob, backup)
|
||||
case strategyv1alpha1.KubernetesStrategyKind:
|
||||
return r.reconcileKubernetesRestore(ctx, restoreJob, backup)
|
||||
default:
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind))
|
||||
}
|
||||
@@ -382,6 +384,12 @@ func (r *RestoreJobReconciler) cleanupOnDelete(ctx context.Context, restoreJob *
|
||||
case strategyv1alpha1.VeleroStrategyKind:
|
||||
r.cleanupVeleroRestore(ctx, restoreJob)
|
||||
|
||||
case strategyv1alpha1.KubernetesStrategyKind:
|
||||
// Its Velero Restores are labelled like the Velero driver's, so the
|
||||
// same cleanup reaps them; a live cluster it stopped is resumed.
|
||||
r.cleanupVeleroRestore(ctx, restoreJob)
|
||||
r.cleanupKubernetesRestore(ctx, restoreJob)
|
||||
|
||||
case strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.JobStrategyKind, strategyv1alpha1.AltinityStrategyKind, strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MongoDBStrategyKind, strategyv1alpha1.FoundationDBStrategyKind, strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind:
|
||||
// Nothing to clean up: these drivers don't materialise namespaced
|
||||
// artifacts that outlive the RestoreJob. (Etcd: the operator-side
|
||||
|
||||
@@ -6,9 +6,9 @@ include ../../../hack/package.mk
|
||||
|
||||
# PHONY so a stray file/dir named `image` cannot mark the aggregate up to date
|
||||
# and skip both builds.
|
||||
.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client
|
||||
.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup
|
||||
|
||||
image: image-backupstrategy-controller image-redis-backup image-backup-http-client
|
||||
image: image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup
|
||||
|
||||
image-backupstrategy-controller:
|
||||
docker buildx build -f images/backupstrategy-controller/Dockerfile ../../.. \
|
||||
@@ -46,5 +46,16 @@ image-backup-http-client: image-redis-backup
|
||||
yq -i '.backupStrategyController.chBackupClientImage = strenv(IMAGE) | .backupStrategyController.rabbitmqBackupClientImage = strenv(IMAGE)' values.yaml
|
||||
rm -f images/backup-http-client.json
|
||||
|
||||
# Chained after image-backup-http-client: one `yq -i` writer at a time.
|
||||
image-kubernetes-backup: image-backup-http-client
|
||||
docker buildx build -f images/kubernetes-backup/Dockerfile images/kubernetes-backup \
|
||||
$(call image-tags,kubernetes-backup,v$(COZYSTACK_VERSION)) \
|
||||
$(call cache-args,kubernetes-backup) \
|
||||
--metadata-file images/kubernetes-backup.json \
|
||||
$(BUILDX_ARGS)
|
||||
IMAGE="$(REGISTRY)/kubernetes-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubernetes-backup.json -o json -r)" \
|
||||
yq -i '.backupStrategyController.kubernetesBackupClientImage = strenv(IMAGE)' values.yaml
|
||||
rm -f images/kubernetes-backup.json
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
Generated
+147
@@ -0,0 +1,147 @@
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.16.4
|
||||
name: kubernetes.strategy.backups.cozystack.io
|
||||
spec:
|
||||
group: strategy.backups.cozystack.io
|
||||
names:
|
||||
kind: Kubernetes
|
||||
listKind: KubernetesList
|
||||
plural: kubernetes
|
||||
singular: kubernetes
|
||||
scope: Cluster
|
||||
versions:
|
||||
- name: v1alpha1
|
||||
schema:
|
||||
openAPIV3Schema:
|
||||
description: |-
|
||||
Kubernetes defines a backup strategy for a tenant Kubernetes application.
|
||||
A tenant cluster keeps its state in two places the platform can reach: the
|
||||
cluster's key prefix in the tenant's shared etcd DataStore, and host-side
|
||||
objects (control-plane PKI Secrets, the application HelmReleases, and the
|
||||
DataVolumes that back the tenant's PersistentVolumes). The strategy captures
|
||||
the first with a one-shot Job and the second with Velero. The two are taken
|
||||
one after another, so a backup is crash-consistent at best.
|
||||
properties:
|
||||
apiVersion:
|
||||
description: |-
|
||||
APIVersion defines the versioned schema of this representation of an object.
|
||||
Servers should convert recognized schemas to the latest internal value, and
|
||||
may reject unrecognized values.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||
type: string
|
||||
kind:
|
||||
description: |-
|
||||
Kind is a string value representing the REST resource this object represents.
|
||||
Servers may infer this from the endpoint the client submits requests to.
|
||||
Cannot be updated.
|
||||
In CamelCase.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||
type: string
|
||||
metadata:
|
||||
type: object
|
||||
spec:
|
||||
description: KubernetesSpec specifies the desired behavior of a Kubernetes
|
||||
backup strategy.
|
||||
properties:
|
||||
certificateWarningThreshold:
|
||||
description: |-
|
||||
CertificateWarningThreshold is how close to expiry a control-plane
|
||||
certificate may be before a backup records a warning. Defaults to 720h.
|
||||
type: string
|
||||
template:
|
||||
description: |-
|
||||
Template holds a PodTemplateSpec for the Job that dumps the cluster's
|
||||
key prefix from its etcd DataStore to object storage, writes it back on
|
||||
restore, and deletes the object on cleanup. The Job runs in the
|
||||
application namespace. Helm-like Go templates are supported in every
|
||||
string field and are rendered against:
|
||||
- `.Release.Name` and `.Release.Namespace` — the application's name and
|
||||
namespace; `.Release.Prefix` is `kubernetes-`.
|
||||
- `.Mode` — `"backup"`, `"restore"` or `"cleanup"`.
|
||||
- `.ObjectKey` — the object-storage key of this backup's dump.
|
||||
- `.Datastore.Prefix` — the cluster's key prefix, without slashes.
|
||||
- `.Datastore.Endpoints` — comma-separated etcd client URLs.
|
||||
- `.Datastore.CertificateSecret` — the Secret holding the cluster's
|
||||
etcd client certificate (`ca.crt`, `server.crt`, `server.key`).
|
||||
- `.Parameters` — the resolved BackupClass strategy parameters.
|
||||
|
||||
Stored schemaless for the same CRD-size reason as redis_types.go.
|
||||
x-kubernetes-preserve-unknown-fields: true
|
||||
velero:
|
||||
description: |-
|
||||
Velero holds the Velero Backup and Restore specs for the host-side
|
||||
objects and volumes. Rendered against `.Application` and `.Parameters`,
|
||||
like the Velero strategy.
|
||||
x-kubernetes-preserve-unknown-fields: true
|
||||
required:
|
||||
- template
|
||||
- velero
|
||||
type: object
|
||||
status:
|
||||
properties:
|
||||
conditions:
|
||||
items:
|
||||
description: Condition contains details for one aspect of the current
|
||||
state of this API Resource.
|
||||
properties:
|
||||
lastTransitionTime:
|
||||
description: |-
|
||||
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||
format: date-time
|
||||
type: string
|
||||
message:
|
||||
description: |-
|
||||
message is a human readable message indicating details about the transition.
|
||||
This may be an empty string.
|
||||
maxLength: 32768
|
||||
type: string
|
||||
observedGeneration:
|
||||
description: |-
|
||||
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||
with respect to the current state of the instance.
|
||||
format: int64
|
||||
minimum: 0
|
||||
type: integer
|
||||
reason:
|
||||
description: |-
|
||||
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||
Producers of specific condition types may define expected values and meanings for this field,
|
||||
and whether the values are considered a guaranteed API.
|
||||
The value should be a CamelCase string.
|
||||
This field may not be empty.
|
||||
maxLength: 1024
|
||||
minLength: 1
|
||||
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||
type: string
|
||||
status:
|
||||
description: status of the condition, one of True, False, Unknown.
|
||||
enum:
|
||||
- "True"
|
||||
- "False"
|
||||
- Unknown
|
||||
type: string
|
||||
type:
|
||||
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||
maxLength: 316
|
||||
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||
type: string
|
||||
required:
|
||||
- lastTransitionTime
|
||||
- message
|
||||
- reason
|
||||
- status
|
||||
- type
|
||||
type: object
|
||||
type: array
|
||||
type: object
|
||||
type: object
|
||||
served: true
|
||||
storage: true
|
||||
subresources:
|
||||
status: {}
|
||||
@@ -0,0 +1,14 @@
|
||||
# kubernetes-backup runs the Kubernetes strategy Pod: it dumps a tenant
|
||||
# cluster's key prefix from its etcd DataStore with etcdctl, moves the dump to
|
||||
# S3 with curl --aws-sigv4, and writes it back with etcdctl on restore. etcdctl
|
||||
# matches the etcd the platform deploys. Everything is installed at build time
|
||||
# so the Pods never run `apk add` and work on air-gapped clusters.
|
||||
FROM quay.io/coreos/etcd:v3.6.11@sha256:6ae247c7666ceec554c51ba1f9bc8dd2212f975370dbd65710c9ca0e36ae1fff AS etcd
|
||||
|
||||
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
|
||||
|
||||
RUN apk add --no-cache curl jq ca-certificates coreutils
|
||||
|
||||
COPY --from=etcd /usr/local/bin/etcdctl /usr/local/bin/etcdctl
|
||||
|
||||
USER 65532:65532
|
||||
@@ -78,6 +78,13 @@ spec:
|
||||
apiGroup: strategy.backups.cozystack.io
|
||||
kind: Redis
|
||||
name: cozy-default-redis
|
||||
- application:
|
||||
apiGroup: apps.cozystack.io
|
||||
kind: Kubernetes
|
||||
strategyRef:
|
||||
apiGroup: strategy.backups.cozystack.io
|
||||
kind: Kubernetes
|
||||
name: cozy-default-kubernetes
|
||||
- application:
|
||||
apiGroup: apps.cozystack.io
|
||||
kind: Kafka
|
||||
|
||||
@@ -62,7 +62,25 @@ rules:
|
||||
# through the dynamic client, so no list/watch)
|
||||
- apiGroups: ["cdi.kubevirt.io"]
|
||||
resources: ["datavolumes"]
|
||||
verbs: ["get", "delete"]
|
||||
verbs: ["get", "list", "delete"]
|
||||
# Kubernetes strategy: read where a tenant cluster keeps its etcd prefix, and
|
||||
# stop and resume its control plane, worker pools and cluster-autoscaler around
|
||||
# an in-place restore.
|
||||
- apiGroups: ["kamaji.clastix.io"]
|
||||
resources: ["tenantcontrolplanes", "datastores"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["controlplane.cluster.x-k8s.io"]
|
||||
resources: ["kamajicontrolplanes"]
|
||||
verbs: ["get", "patch"]
|
||||
- apiGroups: ["cluster.x-k8s.io"]
|
||||
resources: ["machinedeployments"]
|
||||
verbs: ["get", "list", "patch"]
|
||||
- apiGroups: ["cluster.x-k8s.io"]
|
||||
resources: ["machines"]
|
||||
verbs: ["list", "patch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["get", "patch"]
|
||||
# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR
|
||||
# and deletes old. patch additionally lets the DefaultObjectsGate stamp
|
||||
# reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
{{/*
|
||||
Two template layers, as in strategy-redis-default.yaml: Helm renders this
|
||||
file, then the strategy controller renders the resulting CR per BackupJob.
|
||||
Literal "{{ .Foo }}" for the second layer goes through printf.
|
||||
|
||||
Gated on both a Velero BSL and a resolved $bucketName: the Velero half of
|
||||
the backup goes to the BSL and the etcd dump to the bucket. Also gated on
|
||||
its CRD being established, for the upgrade of an existing install that
|
||||
first ships this kind: see strategy-mongodb-default.yaml.
|
||||
*/}}
|
||||
{{- $bucketName := include "backupstrategy-controller.bucketName" . -}}
|
||||
{{- $crdEstablished := include "backupstrategy-controller.crdEstablished" (dict "root" . "crd" "kubernetes.strategy.backups.cozystack.io") -}}
|
||||
{{- if and .Values.velero.bslEnabled $bucketName $crdEstablished -}}
|
||||
apiVersion: strategy.backups.cozystack.io/v1alpha1
|
||||
kind: Kubernetes
|
||||
metadata:
|
||||
name: cozy-default-kubernetes
|
||||
annotations:
|
||||
backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
activeDeadlineSeconds: {{ int .Values.backupStrategyController.kubernetesBackupActiveDeadlineSeconds }}
|
||||
volumes:
|
||||
{{- /*
|
||||
The cluster's own etcd client certificate, the one its kube-apiserver
|
||||
uses. Optional so a missing Secret fails the script with a clear cause
|
||||
instead of wedging the Pod on FailedMount.
|
||||
*/}}
|
||||
- name: etcd-client
|
||||
secret:
|
||||
secretName: {{ printf "{{ .Datastore.CertificateSecret }}" | quote }}
|
||||
optional: true
|
||||
{{- if .Values.backupStorage.endpointCASecretName }}
|
||||
- name: s3-ca
|
||||
secret:
|
||||
secretName: {{ .Values.backupStorage.endpointCASecretName | quote }}
|
||||
optional: true
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: kubernetes-backup
|
||||
image: "{{ .Values.backupStrategyController.kubernetesBackupClientImage }}"
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- name: etcd-client
|
||||
mountPath: /etc/etcd-client
|
||||
readOnly: true
|
||||
{{- if .Values.backupStorage.endpointCASecretName }}
|
||||
- name: s3-ca
|
||||
mountPath: /etc/s3-ca
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
env:
|
||||
- name: ETCDCTL_ENDPOINTS
|
||||
value: {{ printf "{{ .Datastore.Endpoints }}" | quote }}
|
||||
- name: S3_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cozy-backups-creds
|
||||
key: endpoint
|
||||
optional: true
|
||||
- name: S3_BUCKET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cozy-backups-creds
|
||||
key: bucketName
|
||||
optional: true
|
||||
- name: S3_REGION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cozy-backups-creds
|
||||
key: region
|
||||
optional: true
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cozy-backups-creds
|
||||
key: AWS_ACCESS_KEY_ID
|
||||
optional: true
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cozy-backups-creds
|
||||
key: AWS_SECRET_ACCESS_KEY
|
||||
optional: true
|
||||
- name: S3_INSECURE_SKIP_VERIFY
|
||||
value: {{ .Values.backupStorage.insecureSkipTLSVerify | quote }}
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -euo pipefail
|
||||
: "${S3_ENDPOINT:?missing endpoint in cozy-backups-creds}"
|
||||
: "${S3_BUCKET:?missing bucketName in cozy-backups-creds}"
|
||||
: "${S3_REGION:?missing region in cozy-backups-creds}"
|
||||
: "${AWS_ACCESS_KEY_ID:?missing AWS_ACCESS_KEY_ID in cozy-backups-creds}"
|
||||
: "${AWS_SECRET_ACCESS_KEY:?missing AWS_SECRET_ACCESS_KEY in cozy-backups-creds}"
|
||||
|
||||
MODE="{{ printf "{{ .Mode }}" }}"
|
||||
KEY="{{ printf "{{ .ObjectKey }}" }}"
|
||||
SCHEMA="{{ printf "{{ .Datastore.Prefix }}" }}"
|
||||
SOURCE_SCHEMA="{{ printf "{{ .Datastore.SourcePrefix }}" }}"
|
||||
|
||||
case "${S3_ENDPOINT}" in
|
||||
http://*|https://*) BASE="${S3_ENDPOINT}" ;;
|
||||
*) BASE="https://${S3_ENDPOINT}" ;;
|
||||
esac
|
||||
TLS=""
|
||||
case "${BASE}" in
|
||||
https://*)
|
||||
if [ -f /etc/s3-ca/ca.crt ]; then TLS="--cacert /etc/s3-ca/ca.crt";
|
||||
elif [ -f /etc/s3-ca/tls.crt ]; then TLS="--cacert /etc/s3-ca/tls.crt";
|
||||
elif [ "${S3_INSECURE_SKIP_VERIFY:-false}" = "true" ]; then TLS="-k";
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
s3() {
|
||||
curl -fsS ${TLS} \
|
||||
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
|
||||
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@"
|
||||
}
|
||||
OBJ="${BASE}/${S3_BUCKET}/${KEY}"
|
||||
{{- /*
|
||||
Where a restore keeps the prefix it replaces. One per backup, so
|
||||
cleanup removes it with the dump. A restore writes it only when it
|
||||
is absent and deletes it on success, so after a failed restore it
|
||||
holds the prefix from before the first failed attempt and a retry
|
||||
cannot overwrite it with a partial one.
|
||||
*/}}
|
||||
PRE_KEY="${KEY}.pre-restore"
|
||||
PRE_OBJ="${BASE}/${S3_BUCKET}/${PRE_KEY}"
|
||||
|
||||
if [ "${MODE}" = "cleanup" ]; then
|
||||
for o in "${OBJ}" "${PRE_OBJ}"; do
|
||||
echo "deleting ${o#"${BASE}/"}"
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \
|
||||
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
|
||||
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
|
||||
-X DELETE "${o}")
|
||||
case "${code}" in
|
||||
2??|404) echo "deleted (HTTP ${code})" ;;
|
||||
*) echo "delete failed: HTTP ${code}" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${MODE}" = "verify" ]; then
|
||||
s3 -I "${OBJ}" > /dev/null
|
||||
echo "s3://${S3_BUCKET}/${KEY} exists"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
{{- /*
|
||||
The prefix comes from the TenantControlPlane: Kamaji's schema is the
|
||||
TenantControlPlane's UID, or <namespace>_<name> for one created by
|
||||
an older Kamaji. Refuse anything else: an empty or slash-bearing
|
||||
value would turn the restore's `del --prefix` into a wipe of every
|
||||
cluster in the shared etcd.
|
||||
*/}}
|
||||
for s in "${SCHEMA}" "${SOURCE_SCHEMA}"; do
|
||||
case "${s}" in
|
||||
""|*[!a-z0-9_-]*) echo "refusing unexpected datastore prefix '${s}'" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
PREFIX="/${SCHEMA}/"
|
||||
{{- /*
|
||||
The prefix the dump was taken under. A rebuilt cluster has a new
|
||||
TenantControlPlane and so a new schema; its keys are written under
|
||||
the new prefix.
|
||||
*/}}
|
||||
SOURCE_PREFIX="/${SOURCE_SCHEMA}/"
|
||||
for f in ca.crt server.crt server.key; do
|
||||
[ -f "/etc/etcd-client/${f}" ] || { echo "etcd client certificate is missing /etc/etcd-client/${f}" >&2; exit 1; }
|
||||
done
|
||||
export ETCDCTL_CACERT=/etc/etcd-client/ca.crt
|
||||
export ETCDCTL_CERT=/etc/etcd-client/server.crt
|
||||
export ETCDCTL_KEY=/etc/etcd-client/server.key
|
||||
|
||||
if [ "${MODE}" = "backup" ]; then
|
||||
etcdctl get "${PREFIX}" --prefix -w json > /tmp/dump.json
|
||||
count=$(jq '.kvs // [] | length' /tmp/dump.json)
|
||||
if [ "${count}" -eq 0 ]; then
|
||||
echo "etcd prefix ${PREFIX} is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "dumped ${count} keys under ${PREFIX}; uploading s3://${S3_BUCKET}/${KEY}"
|
||||
s3 -X PUT --upload-file /tmp/dump.json "${OBJ}"
|
||||
elif [ "${MODE}" = "restore" ]; then
|
||||
{{- /*
|
||||
Everything that can fail without touching etcd runs before the
|
||||
prefix is deleted: the download, the check that every key belongs
|
||||
to this cluster, and a copy of the prefix as it is now, which is
|
||||
what an operator restores from if a failure comes after the delete.
|
||||
*/}}
|
||||
echo "downloading s3://${S3_BUCKET}/${KEY}"
|
||||
s3 -o /tmp/dump.json "${OBJ}"
|
||||
jq -e '.kvs | length > 0' /tmp/dump.json >/dev/null || { echo "dump holds no keys" >&2; exit 1; }
|
||||
jq -e --arg p "${SOURCE_PREFIX}" 'all(.kvs[]; .key | @base64d | startswith($p))' /tmp/dump.json >/dev/null \
|
||||
|| { echo "dump holds keys outside ${SOURCE_PREFIX}" >&2; exit 1; }
|
||||
{{- /*
|
||||
etcdctl put cannot write an empty value, so a dump holding one
|
||||
must stop here rather than halfway through the write.
|
||||
*/}}
|
||||
jq -e 'all(.kvs[]; (.value // "") != "")' /tmp/dump.json >/dev/null \
|
||||
|| { echo "dump holds a key with an empty value, which etcdctl cannot write" >&2; exit 1; }
|
||||
jq -r '.kvs[] | [.key, .value] | @tsv' /tmp/dump.json > /tmp/kvs.tsv
|
||||
{{- /*
|
||||
Only a definite answer decides about the copy: an existing one is
|
||||
kept, a missing one is taken, and any other reply (a 5xx, a
|
||||
timeout) stops the restore while the prefix is still intact,
|
||||
rather than overwriting the copy an earlier failed restore left.
|
||||
*/}}
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -I ${TLS} \
|
||||
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
|
||||
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
|
||||
"${PRE_OBJ}") || code=000
|
||||
case "${code}" in
|
||||
200) echo "keeping s3://${S3_BUCKET}/${PRE_KEY} from an earlier failed restore" ;;
|
||||
404)
|
||||
etcdctl get "${PREFIX}" --prefix -w json > /tmp/before.json
|
||||
echo "saving the current prefix to s3://${S3_BUCKET}/${PRE_KEY}"
|
||||
s3 -X PUT --upload-file /tmp/before.json "${PRE_OBJ}"
|
||||
;;
|
||||
*) echo "cannot tell whether s3://${S3_BUCKET}/${PRE_KEY} exists: HTTP ${code}" >&2; exit 1 ;;
|
||||
esac
|
||||
{{- /*
|
||||
Delete first so keys created after the backup do not survive the
|
||||
restore. Events and master leases are bound to etcd leases that
|
||||
did not survive; written back without one they would never expire,
|
||||
so they are skipped. They are matched as whole top-level resources
|
||||
under the prefix, not as a path segment anywhere, which would also
|
||||
drop every object in a namespace named "events".
|
||||
*/}}
|
||||
{{- /*
|
||||
The termination message tells the driver that this Pod got as far
|
||||
as changing the prefix: only then is the control plane left
|
||||
stopped after a failure.
|
||||
*/}}
|
||||
echo writing > /dev/termination-log
|
||||
etcdctl del "${PREFIX}" --prefix
|
||||
n=0
|
||||
while IFS="$(printf '\t')" read -r k v; do
|
||||
key=$(printf '%s' "${k}" | base64 -d)
|
||||
rel="${key#"${SOURCE_PREFIX}"}"
|
||||
case "${rel}" in events/*|masterleases/*) continue ;; esac
|
||||
printf '%s' "${v}" | base64 -d > /tmp/value
|
||||
etcdctl put "${PREFIX}${rel}" < /tmp/value > /dev/null
|
||||
n=$((n + 1))
|
||||
done < /tmp/kvs.tsv
|
||||
echo "restored ${n} keys under ${PREFIX}"
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \
|
||||
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
|
||||
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
|
||||
-X DELETE "${PRE_OBJ}") || code=000
|
||||
case "${code}" in
|
||||
2??|404) echo "dropped s3://${S3_BUCKET}/${PRE_KEY}" ;;
|
||||
*) echo "WARNING: could not drop s3://${S3_BUCKET}/${PRE_KEY} (HTTP ${code}); a later restore would keep it as the pre-restore copy, so delete it by hand" >&2 ;;
|
||||
esac
|
||||
else
|
||||
echo "unknown mode: ${MODE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
resources:
|
||||
{{- toYaml .Values.backupStrategyController.kubernetesBackupResources | nindent 12 }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
velero:
|
||||
{{- /*
|
||||
Secrets (control-plane and Talos PKI, by the Kamaji and Helm instance
|
||||
labels) and the DataVolumes and PVCs backing the cluster's
|
||||
PersistentVolumes (the CSI driver stamps the cluster-name label on them;
|
||||
worker system disks do not carry it). The HelmReleases are recorded by the
|
||||
driver on the Backup instead: selecting them by application label would
|
||||
also select worker disks the lineage webhook labels the same way. The
|
||||
driver sets the restore's resources and selectors per phase.
|
||||
*/}}
|
||||
spec:
|
||||
includedNamespaces:
|
||||
- {{ printf "{{ .Application.metadata.namespace }}" | quote }}
|
||||
orLabelSelectors:
|
||||
- matchLabels:
|
||||
kamaji.clastix.io/name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }}
|
||||
- matchLabels:
|
||||
app.kubernetes.io/instance: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }}
|
||||
- matchLabels:
|
||||
cluster.x-k8s.io/cluster-name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }}
|
||||
includedResources:
|
||||
- secrets
|
||||
- persistentvolumeclaims
|
||||
- datavolumes.cdi.kubevirt.io
|
||||
includeClusterResources: false
|
||||
storageLocation: cozy-default
|
||||
snapshotVolumes: true
|
||||
snapshotMoveData: true
|
||||
ttl: 720h0m0s
|
||||
itemOperationTimeout: 24h0m0s
|
||||
restoreSpec:
|
||||
includeClusterResources: false
|
||||
{{- end -}}
|
||||
@@ -22,6 +22,7 @@ templates:
|
||||
- templates/strategy-foundationdb-default.yaml
|
||||
- templates/strategy-velero-vminstance-default.yaml
|
||||
- templates/strategy-velero-vmdisk-default.yaml
|
||||
- templates/strategy-kubernetes-default.yaml
|
||||
|
||||
release:
|
||||
name: t
|
||||
@@ -88,11 +89,14 @@ tests:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
template: templates/strategy-velero-vmdisk-default.yaml
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
template: templates/strategy-kubernetes-default.yaml
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
template: templates/velero-bsl.yaml
|
||||
|
||||
- it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all eleven routes (FoundationDB intentionally unbound)"
|
||||
- it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all twelve routes (FoundationDB intentionally unbound)"
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
@@ -103,7 +107,18 @@ tests:
|
||||
template: templates/backupclass-default.yaml
|
||||
- lengthEqual:
|
||||
path: spec.strategies
|
||||
count: 11
|
||||
count: 12
|
||||
template: templates/backupclass-default.yaml
|
||||
- contains:
|
||||
path: spec.strategies
|
||||
content:
|
||||
application:
|
||||
apiGroup: apps.cozystack.io
|
||||
kind: Kubernetes
|
||||
strategyRef:
|
||||
apiGroup: strategy.backups.cozystack.io
|
||||
kind: Kubernetes
|
||||
name: cozy-default-kubernetes
|
||||
template: templates/backupclass-default.yaml
|
||||
- contains:
|
||||
path: spec.strategies
|
||||
@@ -240,6 +255,18 @@ tests:
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
template: templates/strategy-velero-vmdisk-default.yaml
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
template: templates/strategy-kubernetes-default.yaml
|
||||
# The driver renders these per BackupJob; Helm must leave them literal.
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].secret.secretName
|
||||
value: "{{ .Datastore.CertificateSecret }}"
|
||||
template: templates/strategy-kubernetes-default.yaml
|
||||
- equal:
|
||||
path: spec.velero.spec.orLabelSelectors[2].matchLabels["cluster.x-k8s.io/cluster-name"]
|
||||
value: "kubernetes-{{ .Application.metadata.name }}"
|
||||
template: templates/strategy-kubernetes-default.yaml
|
||||
- hasDocuments:
|
||||
count: 1
|
||||
template: templates/velero-bsl.yaml
|
||||
@@ -347,6 +374,7 @@ tests:
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-foundationdb-default.yaml}
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml}
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml}
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml}
|
||||
# The ungated BackupClass keeps rendering, which is what makes the counts
|
||||
# above an assertion about the CRD gate rather than about a chart-wide
|
||||
# render failure.
|
||||
@@ -432,6 +460,7 @@ tests:
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
@@ -449,6 +478,7 @@ tests:
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-redis-default.yaml}
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml}
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml}
|
||||
- {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml}
|
||||
|
||||
- it: "with every CRD Established and the BucketClaim resolved, every Strategy CR renders against the COSI bucket name"
|
||||
# The positive half of the case above, through the same live lookups and no
|
||||
@@ -471,6 +501,7 @@ tests:
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
|
||||
@@ -489,6 +520,7 @@ tests:
|
||||
- {hasDocuments: {count: 1}, template: templates/strategy-redis-default.yaml}
|
||||
- {hasDocuments: {count: 1}, template: templates/strategy-velero-vminstance-default.yaml}
|
||||
- {hasDocuments: {count: 1}, template: templates/strategy-velero-vmdisk-default.yaml}
|
||||
- {hasDocuments: {count: 1}, template: templates/strategy-kubernetes-default.yaml}
|
||||
|
||||
- it: "velero.bslEnabled=false: Velero strategies and BSL drop out while other strategies keep rendering"
|
||||
set:
|
||||
|
||||
@@ -13,11 +13,11 @@ release:
|
||||
namespace: cozy-backup-controller
|
||||
|
||||
tests:
|
||||
- it: "ClusterRole can read and delete DataVolumes"
|
||||
- it: "ClusterRole can read, list and delete DataVolumes"
|
||||
asserts:
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: ["cdi.kubevirt.io"]
|
||||
resources: ["datavolumes"]
|
||||
verbs: ["get", "delete"]
|
||||
verbs: ["get", "list", "delete"]
|
||||
@@ -37,6 +37,24 @@ backupStrategyController:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 4Gi
|
||||
# kubernetesBackupClientImage is rendered into the Kubernetes strategy Pod,
|
||||
# which dumps a tenant cluster's etcd prefix with etcdctl and moves it to S3
|
||||
# with curl. Built from images/kubernetes-backup/; `make image` re-digests it.
|
||||
kubernetesBackupClientImage: "ghcr.io/cozystack/cozystack/kubernetes-backup:v1.6.0"
|
||||
# kubernetesBackupActiveDeadlineSeconds bounds one etcd dump/restore Job.
|
||||
kubernetesBackupActiveDeadlineSeconds: 3600
|
||||
# kubernetesBackupResources sizes the Kubernetes strategy Pod. The dump is held
|
||||
# in ephemeral storage while it moves and jq reads it into memory, so a
|
||||
# cluster with a large etcd prefix needs more of both.
|
||||
kubernetesBackupResources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
ephemeral-storage: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
ephemeral-storage: 2Gi
|
||||
# The Kafka metadata strategy carries no client image here: unlike the Altinity
|
||||
# and RabbitMQ drivers, which need only curl (plus jq for Altinity), its driver
|
||||
# runs the Kafka Admin-API CLI (kafka-topics / kafka-configs), which only the
|
||||
|
||||
Reference in new issue
Block a user