feat(backups): add Kubernetes backup strategy for tenant clusters

A tenant Kubernetes application had no backup route in cozy-default, and
none of the existing strategies fits it. Its API state is a key prefix in
the etcd DataStore the whole tenant shares, so the Etcd strategy's snapshot
would roll every tenant cluster back at once; its control-plane CA, service
account key and Talos secrets cannot be regenerated without breaking every
node and credential issued under them; and its PersistentVolumes are
DataVolumes in the tenant namespace that the application's own deletion
removes.

The new Kubernetes strategy backs up the cluster's etcd prefix with a Job
that authenticates with the cluster's own datastore certificate, and the PKI
Secrets and data DataVolumes with Velero. The HelmReleases of the cluster
and its node pools are recorded on the Backup rather than in Velero,
because a label selector for them would also pick up the worker system
disks. A backup warns when a CA would expire before the backup does.

A restore rebuilds a deleted cluster under its old CA, or rolls a live one
back after stopping its control plane and workers, and lets the etcd
prefix and the volumes be restored separately.

Signed-off-by: Andrey Kolkov <androndo@gmail.com>
Assisted-by: LLM
This commit is contained in:
Andrey Kolkov committed 2026-11-10 15:28:34 +07:00
1 parent 5e9c392352
commit 0f44cf9e25
28 files changed
+4697 -22

No files matched your search

@@ -0,0 +1,95 @@
// SPDX-License-Identifier: Apache-2.0
// Package v1alpha1 defines strategy.backups.cozystack.io API types.
//
// Group: strategy.backups.cozystack.io
// Version: v1alpha1
package v1alpha1
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
func init() {
SchemeBuilder.Register(func(s *runtime.Scheme) error {
s.AddKnownTypes(GroupVersion,
&Kubernetes{},
&KubernetesList{},
)
return nil
})
}
const (
KubernetesStrategyKind = "Kubernetes"
)
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:scope=Cluster
// Kubernetes defines a backup strategy for a tenant Kubernetes application.
// A tenant cluster keeps its state in two places the platform can reach: the
// cluster's key prefix in the tenant's shared etcd DataStore, and host-side
// objects (control-plane PKI Secrets, the application HelmReleases, and the
// DataVolumes that back the tenant's PersistentVolumes). The strategy captures
// the first with a one-shot Job and the second with Velero. The two are taken
// one after another, so a backup is crash-consistent at best.
type Kubernetes struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec KubernetesSpec `json:"spec,omitempty"`
Status KubernetesStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// KubernetesList contains a list of Kubernetes backup strategies.
type KubernetesList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Kubernetes `json:"items"`
}
// KubernetesSpec specifies the desired behavior of a Kubernetes backup strategy.
type KubernetesSpec struct {
// Template holds a PodTemplateSpec for the Job that dumps the cluster's
// key prefix from its etcd DataStore to object storage, writes it back on
// restore, and deletes the object on cleanup. The Job runs in the
// application namespace. Helm-like Go templates are supported in every
// string field and are rendered against:
// - `.Release.Name` and `.Release.Namespace` — the application's name and
// namespace; `.Release.Prefix` is `kubernetes-`.
// - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`.
// - `.ObjectKey` — the object-storage key of this backup's dump.
// - `.Datastore.Prefix` — the cluster's key prefix, without slashes.
// - `.Datastore.Endpoints` — comma-separated etcd client URLs.
// - `.Datastore.CertificateSecret` — the Secret holding the cluster's
// etcd client certificate (`ca.crt`, `server.crt`, `server.key`).
// - `.Parameters` — the resolved BackupClass strategy parameters.
//
// Stored schemaless for the same CRD-size reason as redis_types.go.
//
// +kubebuilder:validation:Schemaless
// +kubebuilder:pruning:PreserveUnknownFields
Template corev1.PodTemplateSpec `json:"template"`
// Velero holds the Velero Backup and Restore specs for the host-side
// objects and volumes. Rendered against `.Application` and `.Parameters`,
// like the Velero strategy.
//
// +kubebuilder:validation:Schemaless
// +kubebuilder:pruning:PreserveUnknownFields
Velero VeleroTemplate `json:"velero"`
// CertificateWarningThreshold is how close to expiry a control-plane
// certificate may be before a backup records a warning. Defaults to 720h.
// +optional
CertificateWarningThreshold *metav1.Duration `json:"certificateWarningThreshold,omitempty"`
}
type KubernetesStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
+103
View File
@@ -839,6 +839,109 @@ func (in *KafkaStatus) DeepCopy() *KafkaStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Kubernetes) DeepCopyInto(out *Kubernetes) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kubernetes.
func (in *Kubernetes) DeepCopy() *Kubernetes {
if in == nil {
return nil
}
out := new(Kubernetes)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *Kubernetes) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KubernetesList) DeepCopyInto(out *KubernetesList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]Kubernetes, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesList.
func (in *KubernetesList) DeepCopy() *KubernetesList {
if in == nil {
return nil
}
out := new(KubernetesList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *KubernetesList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) {
*out = *in
in.Template.DeepCopyInto(&out.Template)
in.Velero.DeepCopyInto(&out.Velero)
if in.CertificateWarningThreshold != nil {
in, out := &in.CertificateWarningThreshold, &out.CertificateWarningThreshold
*out = new(v1.Duration)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec.
func (in *KubernetesSpec) DeepCopy() *KubernetesSpec {
if in == nil {
return nil
}
out := new(KubernetesSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus.
func (in *KubernetesStatus) DeepCopy() *KubernetesStatus {
if in == nil {
return nil
}
out := new(KubernetesStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *MariaDB) DeepCopyInto(out *MariaDB) {
*out = *in
+94
View File
@@ -0,0 +1,94 @@
#!/bin/bash
# Shared helpers for the tenant Kubernetes backup/restore walkthrough.
# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh"
export RED='\033[0;31m'
export GREEN='\033[0;32m'
export YELLOW='\033[1;33m'
export BLUE='\033[0;34m'
export MAGENTA='\033[0;35m'
export CYAN='\033[0;36m'
export NC='\033[0m'
export BOLD='\033[1m'
export NAMESPACE="${NAMESPACE:-tenant-root}"
export CLUSTER_NAME="${CLUSTER_NAME:-backup-demo}"
export POOL_NAME="${POOL_NAME:-md0}"
export WORKER_DISK_SIZE="${WORKER_DISK_SIZE:-20Gi}"
export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-cozy-default}"
export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kubernetes-backup-job}"
export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kubernetes-restore-inplace}"
export RESTOREJOB_RECREATE_NAME="${RESTOREJOB_RECREATE_NAME:-kubernetes-restore-recreate}"
export TENANT_API_PORT="${TENANT_API_PORT:-16443}"
export MARKER_VALUE="${MARKER_VALUE:-roundtrip-7c2e91d4}"
export RELEASE="kubernetes-${CLUSTER_NAME}"
log_info() { echo -e "${BLUE}i${NC} $*" >&2; }
log_success() { echo -e "${GREEN}OK${NC} $*" >&2; }
log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; }
log_error() { echo -e "${RED}x${NC} $*" >&2; }
log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; }
print_header() {
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
}
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# The admin kubeconfig points at the in-cluster Service, so the walkthrough
# reaches the tenant API through a port-forward to it and keeps the TLS server
# name the certificate is issued for.
TENANT_KUBECONFIG="$(dirname "${BASH_SOURCE[0]}")/.tenant-kubeconfig"
export TENANT_KUBECONFIG
tenant_connect() {
kubectl -n "$NAMESPACE" get secret "${RELEASE}-admin-kubeconfig" \
-o jsonpath='{.data.super-admin\.svc}' | base64 -d > "$TENANT_KUBECONFIG"
chmod 600 "$TENANT_KUBECONFIG"
local cluster
cluster=$(kubectl --kubeconfig "$TENANT_KUBECONFIG" config view -o jsonpath='{.clusters[0].name}')
kubectl --kubeconfig "$TENANT_KUBECONFIG" config set-cluster "$cluster" \
--server="https://127.0.0.1:${TENANT_API_PORT}" \
--tls-server-name="${RELEASE}.${NAMESPACE}.svc" >/dev/null
pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true
kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}" "${TENANT_API_PORT}:6443" >/dev/null 2>&1 &
for _ in $(seq 1 30); do
tkubectl get --raw /readyz >/dev/null 2>&1 && return 0
sleep 2
done
log_error "tenant API of ${RELEASE} is not reachable through the port-forward"
return 1
}
tenant_disconnect() {
pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true
}
tkubectl() {
kubectl --kubeconfig "$TENANT_KUBECONFIG" "$@"
}
# The CA fingerprint is what proves a restore brought back the cluster rather
# than a new one: nodes, kubeconfigs and tokens all trust this CA.
cluster_ca_fingerprint() {
kubectl -n "$NAMESPACE" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' \
| base64 -d | openssl x509 -noout -fingerprint -sha256 | cut -d= -f2
}
# wait_tenant_marker checks both halves of the backup: the ConfigMap comes back
# with etcd, the file on the PVC with the volumes.
wait_tenant_marker() {
local timeout="${1:-900}" elapsed=0 cm data
while true; do
cm=$(tkubectl -n demo get configmap marker -o jsonpath='{.data.marker}' 2>/dev/null || true)
data=$(tkubectl -n demo exec deploy/writer -- cat /data/marker 2>/dev/null || true)
if [[ "$cm" == "$MARKER_VALUE" && "$data" == "$MARKER_VALUE" ]]; then
log_success "marker present in etcd (ConfigMap) and on the volume"
return 0
fi
(( elapsed >= timeout )) && { log_error "marker not restored: configmap='${cm}' volume='${data}'"; return 1; }
sleep 10
elapsed=$((elapsed + 10))
done
}
+102
View File
@@ -0,0 +1,102 @@
#!/bin/bash
# Step 01: Create a tenant Kubernetes cluster with one worker pool and seed it
# with a marker held in etcd (a ConfigMap) and on a PersistentVolume.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 01: Create Kubernetes '${CLUSTER_NAME}' and seed a marker"
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Kubernetes
metadata:
name: ${CLUSTER_NAME}
namespace: ${NAMESPACE}
spec:
storageClass: replicated
---
apiVersion: apps.cozystack.io/v1alpha1
kind: KubernetesNodes
metadata:
name: ${CLUSTER_NAME}-${POOL_NAME}
namespace: ${NAMESPACE}
spec:
cluster: ${CLUSTER_NAME}
minReplicas: 1
maxReplicas: 1
instanceType: u1.medium
diskSize: ${WORKER_DISK_SIZE}
EOF
wait_hr_ready "$RELEASE" 1200
log_substep "Waiting for the worker node to join..."
tenant_connect
kubectl_ok=0
for _ in $(seq 1 120); do
if tkubectl wait --for=condition=Ready node --all --timeout=10s >/dev/null 2>&1 \
&& [[ -n "$(tkubectl get nodes -o name)" ]] \
&& [[ -n "$(tkubectl get storageclass -o name)" ]]; then
kubectl_ok=1
break
fi
sleep 15
done
(( kubectl_ok )) || { log_error "worker node or StorageClass did not appear"; exit 1; }
tkubectl apply -f - <<EOF
apiVersion: v1
kind: Namespace
metadata:
name: demo
---
apiVersion: v1
kind: ConfigMap
metadata:
name: marker
namespace: demo
data:
marker: ${MARKER_VALUE}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: data
namespace: demo
spec:
accessModes: [ReadWriteOnce]
resources:
requests:
storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: writer
namespace: demo
spec:
replicas: 1
selector:
matchLabels: {app: writer}
template:
metadata:
labels: {app: writer}
spec:
containers:
- name: writer
image: busybox:1.36
command: ["sh", "-c", "[ -f /data/marker ] || echo -n ${MARKER_VALUE} > /data/marker; sync; sleep infinity"]
volumeMounts:
- {name: data, mountPath: /data}
volumes:
- name: data
persistentVolumeClaim: {claimName: data}
EOF
wait_tenant_marker 900
cluster_ca_fingerprint > "$SCRIPT_DIR/.ca-fingerprint"
log_success "cluster CA $(cat "$SCRIPT_DIR/.ca-fingerprint")"
tenant_disconnect
echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupjob.sh"
+36
View File
@@ -0,0 +1,36 @@
#!/bin/bash
# Step 02: Back the cluster up through the cozy-default BackupClass. The
# controller dumps the cluster's etcd prefix to the platform bucket with a
# Job, then backs up its control-plane Secrets and volumes with Velero.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 02: Create BackupJob '${BACKUPJOB_NAME}'"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: BackupJob
metadata:
name: ${BACKUPJOB_NAME}
namespace: ${NAMESPACE}
spec:
applicationRef:
apiGroup: apps.cozystack.io
kind: Kubernetes
name: ${CLUSTER_NAME}
backupClassName: ${BACKUPCLASS_NAME}
EOF
wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 1800 Failed
warn=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" \
-o jsonpath='{.status.conditions[?(@.type=="CertificatesExpiring")].status}')
if [[ "$warn" == "True" ]]; then
log_warning "$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" \
-o jsonpath='{.status.conditions[?(@.type=="CertificatesExpiring")].message}')"
fi
log_success "Backup '$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')' is Ready."
echo -e "\n${GREEN}${BOLD}Next:${NC} ./03-restore-in-place.sh"
+44
View File
@@ -0,0 +1,44 @@
#!/bin/bash
# Step 03: Lose the demo namespace inside the live cluster and roll the cluster
# back. The restore stops the control plane and the workers, writes the etcd
# prefix and the volumes back, and returns the cluster to its replica counts.
# Set RESTORE_OPTIONS='{"volumes":false}' (or '{"etcd":false}') to restore one
# half only.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 03: Restore '${CLUSTER_NAME}' in place"
tenant_connect
log_substep "Deleting namespace demo inside the tenant cluster..."
tkubectl delete namespace demo --wait=true --timeout=10m
tenant_disconnect
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')
options="${RESTORE_OPTIONS:-}"
[[ -n "$options" ]] || options='{}'
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
metadata:
name: ${RESTOREJOB_INPLACE_NAME}
namespace: ${NAMESPACE}
spec:
backupRef:
name: ${backup_ref}
options: ${options}
EOF
wait_for_field restorejob "$RESTOREJOB_INPLACE_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 2400 Failed
wait_hr_ready "$RELEASE" 1200
tenant_connect
wait_tenant_marker 1200
[[ "$(cluster_ca_fingerprint)" == "$(cat "$SCRIPT_DIR/.ca-fingerprint")" ]] \
|| { log_error "cluster CA changed across the restore"; exit 1; }
log_success "same cluster CA, marker back in etcd and on the volume"
tenant_disconnect
echo -e "\n${GREEN}${BOLD}Next:${NC} ./04-restore-after-delete.sh"
+39
View File
@@ -0,0 +1,39 @@
#!/bin/bash
# Step 04: Delete the Kubernetes application and its worker pool, which also
# deletes the cluster's etcd prefix and volumes, then rebuild it from the
# backup under the same name and CA.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 04: Delete '${CLUSTER_NAME}' and rebuild it from the backup"
kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found --wait=false
kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --wait=false
wait_deleted helmrelease "$RELEASE" "$NAMESPACE" 1200
wait_deleted tenantcontrolplane "$RELEASE" "$NAMESPACE" 600
log_substep "Removing what the application's deletion leaves behind, so the rebuild proves itself..."
kubectl -n "$NAMESPACE" delete secret "${RELEASE}-talos-ca" "${RELEASE}-talos-tls-cert" "${RELEASE}-admin-kubeconfig" --ignore-not-found
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}')
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
metadata:
name: ${RESTOREJOB_RECREATE_NAME}
namespace: ${NAMESPACE}
spec:
backupRef:
name: ${backup_ref}
EOF
wait_for_field restorejob "$RESTOREJOB_RECREATE_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 2400 Failed
wait_hr_ready "$RELEASE" 1200
tenant_connect
wait_tenant_marker 1500
[[ "$(cluster_ca_fingerprint)" == "$(cat "$SCRIPT_DIR/.ca-fingerprint")" ]] \
|| { log_error "cluster CA changed across the rebuild"; exit 1; }
log_success "cluster rebuilt with the same CA, marker back in etcd and on the volume"
tenant_disconnect
+68
View File
@@ -0,0 +1,68 @@
# Tenant Kubernetes backup and restore
The `Kubernetes` strategy backs up a tenant Kubernetes application in two halves, taken one after the other, so a backup is crash-consistent at best:
- **etcd**: a Job dumps the cluster's key prefix from the tenant's etcd DataStore to the platform bucket. This is the cluster's API state.
- **Velero**: the control-plane and Talos PKI Secrets, and the DataVolumes and PVCs that back the cluster's PersistentVolumes. Worker system disks are left out; workers are recreated and join with the restored PKI.
The Backup also records the HelmReleases of the cluster and of its worker pools. A backup warns, through a `CertificatesExpiring` condition on the BackupJob, when a control-plane CA expires before the backup's TTL runs out or a leaf certificate expires within `certificateWarningThreshold` (720h).
## Admin: prepare
Nothing to do when the platform default backups are on: `cozy-default` routes `Kubernetes` to the `cozy-default-kubernetes` strategy.
## User: back up
```yaml
apiVersion: backups.cozystack.io/v1alpha1
kind: BackupJob
metadata:
name: my-cluster-backup
namespace: tenant-foo
spec:
applicationRef:
apiGroup: apps.cozystack.io
kind: Kubernetes
name: my-cluster
backupClassName: cozy-default
```
A `Plan` with the same `applicationRef` schedules it.
## User: restore
A restore always goes into the application the backup was taken from.
```yaml
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
metadata:
name: my-cluster-restore
namespace: tenant-foo
spec:
backupRef:
name: my-cluster-backup
options:
etcd: true # restore the API state (default)
volumes: true # restore the PersistentVolume data (default)
```
- **Cluster deleted**: the restore rebuilds it. It restores the PKI Secrets and the volumes, recreates the cluster, writes the etcd state under the new control plane's prefix, and then recreates the worker pools, so the cluster comes back on its old state and its old CA. The cluster and pool HelmReleases are recreated with the spec recorded at backup time, so a backup taken before a platform upgrade that changed the application's values brings back the older values, which the upgraded chart may need migrated by hand.
- **Cluster live**: the restore rolls it back. It must be the cluster the backup was taken from: one deleted and created again under the same name has a new CA and service-account key, so the restore refuses it, and the way back to the backed-up cluster is to delete the application and run the restore again as a rebuild. It suspends the cluster's HelmReleases, stops the workers and the cluster-autoscaler when `volumes` are restored and then the control plane when `etcd` is, writes the selected halves back, and returns every replica count it changed. Workers are recreated, so expect the tenant's Pods to restart. Deleting the RestoreJob midway also returns the replica counts, unless its etcd write is under way.
Nothing is changed until the restore has confirmed that the backup's Velero half and etcd dump exist. If the etcd write fails partway, the control plane stays stopped and the RestoreJob points at the copy of the prefix it saved before writing; run a new RestoreJob to retry.
Restoring only one half leaves the other as it is now: `etcd` alone keeps the current volume contents, `volumes` alone keeps the current API objects.
The two halves differ in what a failed restore leaves behind. The etcd half saves the current prefix before it writes and can be retried from that copy. The volume half has no such copy: Velero cannot overwrite an existing claim, so a live-cluster restore deletes the current DataVolumes and PVCs before it restores them, and if the Velero restore then fails, the volume data exists only in the backup.
The etcd round trip scales with the number of keys under the cluster's prefix. The backup Job holds the whole dump in memory while it checks it, and the restore writes one key per `etcdctl` call: on a test cluster a 441-key restore took about 40 seconds including Pod start. At that rate the default `kubernetesBackupActiveDeadlineSeconds` (1h) covers a prefix of a few tens of thousands of keys. For a cluster with a larger API state, raise it together with the memory limit in `kubernetesBackupResources`.
## Walkthrough
```bash
./run-all.sh # create, back up, roll back in place, rebuild after delete
./cleanup.sh
```
`NAMESPACE`, `CLUSTER_NAME` and `WORKER_DISK_SIZE` are read from the environment; see `00-helpers.sh`.
+18
View File
@@ -0,0 +1,18 @@
#!/bin/bash
# Remove everything the walkthrough created. Deleting the BackupJob's Backup
# removes its Velero backup and its etcd dump object.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Cleanup"
tenant_disconnect
kubectl -n "$NAMESPACE" delete restorejob "$RESTOREJOB_INPLACE_NAME" "$RESTOREJOB_RECREATE_NAME" --ignore-not-found
backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}' 2>/dev/null || true)
kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found
[[ -n "$backup_ref" ]] && kubectl -n "$NAMESPACE" delete backup "$backup_ref" --ignore-not-found
kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found
kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --ignore-not-found
rm -f "$TENANT_KUBECONFIG" "$SCRIPT_DIR/.ca-fingerprint"
log_success "done"
+29
View File
@@ -0,0 +1,29 @@
#!/bin/bash
# End-to-end tenant Kubernetes backup/restore walkthrough. It proves the
# restore brings back the same cluster: the CA fingerprint is unchanged, and a
# marker kept in etcd and one kept on a PersistentVolume both come back, first
# after rolling a live cluster back, then after rebuilding a deleted one.
#
# Needs the platform default backups (backupstrategy-controller with a Velero
# BSL and the system bucket), so cozy-default routes Kubernetes to
# cozy-default-kubernetes.
#
# Env knobs: see 00-helpers.sh. SKIP_RESTORE=1 stops after the backup.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
kubectl wait --for=create kubernetes.strategy.backups.cozystack.io/cozy-default-kubernetes --timeout=15m >/dev/null \
|| { log_error "cozy-default-kubernetes strategy is missing: enable the platform default backups first"; exit 1; }
steps=(01 02 03 04)
[[ "${SKIP_RESTORE:-0}" == "1" ]] && steps=(01 02)
for step in "${steps[@]}"; do
script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)"
[[ -n "$script" ]] || { echo "no script found for step $step" >&2; exit 1; }
echo "=== Running $(basename "$script") ==="
bash "$script"
done
print_header "Walkthrough complete"
@@ -183,6 +183,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
// deletes the object and WAITS for that delete before the Backup is
// removed, so no object is orphaned (see cleanupKafkaBackup).
return r.cleanupKafkaBackup(ctx, backup)
case strategyv1alpha1.KubernetesStrategyKind:
// Owns two artifacts: the Velero backup and the etcd dump object.
return r.cleanupKubernetesBackup(ctx, backup)
case strategyv1alpha1.VeleroStrategyKind:
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
default:
@@ -214,6 +214,8 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return r.reconcileRedis(ctx, j, resolved)
case strategyv1alpha1.KafkaStrategyKind:
return r.reconcileKafka(ctx, j, resolved)
case strategyv1alpha1.KubernetesStrategyKind:
return r.reconcileKubernetes(ctx, j, resolved)
default:
logger.V(1).Info("BackupJob resolved StrategyRef.Kind not supported, skipping",
"backupjob", j.Name,
@@ -240,6 +242,7 @@ func supportedBackupStrategyKinds() []string {
strategyv1alpha1.RabbitmqStrategyKind,
strategyv1alpha1.RedisStrategyKind,
strategyv1alpha1.KafkaStrategyKind,
strategyv1alpha1.KubernetesStrategyKind,
}
}
@@ -45,6 +45,7 @@ func TestSupportedBackupStrategyKindsMatchesDispatch(t *testing.T) {
strategyv1alpha1.RabbitmqStrategyKind,
strategyv1alpha1.RedisStrategyKind,
strategyv1alpha1.KafkaStrategyKind,
strategyv1alpha1.KubernetesStrategyKind,
}
sort.Strings(got)
sort.Strings(want)
@@ -593,12 +593,13 @@ func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *ba
group = *strategy.StrategyRef.APIGroup
}
// When the Velero BSL is disabled (velero.bslEnabled=false, surfaced
// here as an empty VeleroNamespace) the chart gates the Velero
// Strategy CRs off the SAME flag, so they are never rendered. The
// BackupClass still routes to them unconditionally, so counting them
// as missing would force a Helm upgrade every MinForceInterval
// forever against a render that can never produce them.
if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" {
// here as an empty VeleroNamespace) the chart gates the Strategy CRs
// that back up through Velero (Velero, Kubernetes) off the SAME flag,
// so they are never rendered. The BackupClass still routes to them
// unconditionally, so counting them as missing would force a Helm
// upgrade every MinForceInterval forever against a render that can
// never produce them.
if g.VeleroNamespace == "" && group == strategyAPIGroup && (kind == "Velero" || kind == "Kubernetes") {
continue
}
key := fmt.Sprintf("%s/%s", kind, name)
@@ -37,6 +37,7 @@ var gateGVKs = []schema.GroupVersionKind{
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"},
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"},
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"},
{Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Kubernetes"},
{Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"},
{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"},
}
@@ -109,6 +110,7 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass {
ref("Altinity", "cozy-default-altinity"),
ref("Velero", "cozy-default-velero-vminstance"),
ref("Velero", "cozy-default-velero-vmdisk"),
ref("Kubernetes", "cozy-default-kubernetes"),
},
},
}
@@ -149,6 +151,7 @@ func allDefaultObjects() []runtime.Object {
strategyObject("Altinity", "cozy-default-altinity"),
strategyObject("Velero", "cozy-default-velero-vminstance"),
strategyObject("Velero", "cozy-default-velero-vmdisk"),
strategyObject("Kubernetes", "cozy-default-kubernetes"),
bslObject(),
}
}
@@ -288,8 +291,8 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) {
t.Fatal("expected a forced Helm upgrade when default objects are missing")
}
// 6 strategyRefs (Velero twice under different names) + the BSL.
if len(missing) != 7 {
t.Fatalf("missing = %v, want 7 entries", missing)
if len(missing) != 8 {
t.Fatalf("missing = %v, want 8 entries", missing)
}
ann := forceAnnotations(t, dyn)
forceAt, ok := ann["reconcile.fluxcd.io/forceAt"]
@@ -319,6 +322,7 @@ func TestCheckNoopWhenAllPresent(t *testing.T) {
strategyObject("Altinity", "cozy-default-altinity"),
strategyObject("Velero", "cozy-default-velero-vminstance"),
strategyObject("Velero", "cozy-default-velero-vmdisk"),
strategyObject("Kubernetes", "cozy-default-kubernetes"),
bslObject(),
)
@@ -454,8 +458,8 @@ func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) {
if !forced {
t.Fatal("objects release throttled by the earlier credentials force")
}
if len(missing) != 7 {
t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing)
if len(missing) != 8 {
t.Fatalf("missing = %v, want the 7 strategies + the BSL", missing)
}
if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
t.Errorf("own HelmRelease not forced: %v", ann)
@@ -482,7 +486,7 @@ func TestForceSkipsSuspendedHelmRelease(t *testing.T) {
if forced {
t.Error("forced reported true for a suspended HelmRelease that ignores the annotations")
}
if len(missing) != 7 {
if len(missing) != 8 {
t.Fatalf("missing = %v, want the objects still reported", missing)
}
if ann := forceAnnotations(t, dyn); len(ann) != 0 {
@@ -599,6 +603,7 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) {
strategyObject("Altinity", "cozy-default-altinity"),
strategyObject("Velero", "cozy-default-velero-vminstance"),
strategyObject("Velero", "cozy-default-velero-vmdisk"),
strategyObject("Kubernetes", "cozy-default-kubernetes"),
bslObject(),
)
@@ -658,6 +663,7 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) {
strategyObject("Altinity", "cozy-default-altinity"),
strategyObject("Velero", "cozy-default-velero-vminstance"),
strategyObject("Velero", "cozy-default-velero-vmdisk"),
strategyObject("Kubernetes", "cozy-default-kubernetes"),
)
// Rebuild the RESTMapper without BackupStorageLocation: the velero.io API
// is not served on this cluster.
@@ -1053,8 +1059,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) {
g.lastForce = base.Add(-time.Minute)
g.now = func() time.Time { return base }
if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 {
t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err)
if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != len(allDefaultObjects()) {
t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want %d reported, none forced", missing, forced, err, len(allDefaultObjects()))
}
g.now = func() time.Time { return base.Add(5 * time.Minute) }
@@ -1065,8 +1071,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) {
g.now = func() time.Time { return base.Add(time.Hour) }
missing, forced, err := g.Check(context.Background())
if err != nil || !forced || len(missing) != 7 {
t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err)
if err != nil || !forced || len(missing) != len(allDefaultObjects()) {
t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want %d reported and forced again", missing, forced, err, len(allDefaultObjects()))
}
if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first {
t.Errorf("not forced again for unannotated objects: forceAt still %q", first)
@@ -0,0 +1,862 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"crypto/x509"
"encoding/json"
"encoding/pem"
"fmt"
"maps"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"time"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/template"
)
const (
kubernetesAppKind = "Kubernetes"
kubernetesReleasePrefix = "kubernetes-"
kubernetesLabelMode = "kubernetes.strategy.backups.cozystack.io/mode"
kubernetesModeBackup = "backup"
kubernetesModeRestore = "restore"
kubernetesModeCleanup = "cleanup"
// kubernetesModeVerify checks that the dump object exists before a restore
// changes anything.
kubernetesModeVerify = "verify"
// kubernetesCleanupAttemptsAnnotation counts failed cleanup Jobs on the
// Backup, so a persistently failing delete (revoked credentials, a removed
// bucket) ends in a released Backup and a Warning rather than a Job storm.
kubernetesCleanupAttemptsAnnotation = "kubernetes.strategy.backups.cozystack.io/cleanup-attempts"
kubernetesCleanupMaxAttempts = 5
kubernetesMetaPrefix = "kubernetes.strategy.backups.cozystack.io/"
kubernetesObjectMetaKey = kubernetesMetaPrefix + "object"
kubernetesDatastorePrefixMetaKey = kubernetesMetaPrefix + "datastore-prefix"
kubernetesDatastoreEndpointsMetaKey = kubernetesMetaPrefix + "datastore-endpoints"
kubernetesHelmReleasesMetaKey = kubernetesMetaPrefix + "helmreleases"
kubernetesDataVolumesMetaKey = kubernetesMetaPrefix + "datavolumes"
kubernetesCertWarningsMetaKey = kubernetesMetaPrefix + "certificate-warnings"
kubernetesControlPlaneUIDMetaKey = kubernetesMetaPrefix + "tenantcontrolplane-uid"
kubernetesParamPrefix = kubernetesMetaPrefix + "parameter/"
// kubernetesCondCertificates is set on the BackupJob once the control-plane
// certificates have been checked; Status=True means at least one expires
// inside the warning window.
kubernetesCondCertificates = "CertificatesExpiring"
kubernetesPollInterval = 5 * time.Second
// Leaf certificates are rotated by Kamaji and cert-manager, so a near
// expiry is only worth a warning when it is this close.
kubernetesDefaultCertThreshold = 720 * time.Hour
// A CA must outlive every backup that may still be restored, so its
// window is the Velero backup TTL; this is the fallback when none is set.
kubernetesDefaultBackupTTL = 720 * time.Hour
helmReleaseNameAnnotation = "meta.helm.sh/release-name"
clusterNameLabel = "cluster.x-k8s.io/cluster-name"
helmInstanceLabel = "app.kubernetes.io/instance"
kamajiNameLabel = "kamaji.clastix.io/name"
)
var (
tenantControlPlaneGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "tenantcontrolplanes"}
kamajiDataStoreGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "datastores"}
kamajiControlPlaneGVR = schema.GroupVersionResource{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Resource: "kamajicontrolplanes"}
machineDeploymentGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"}
machineGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machines"}
deploymentGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
)
// kubernetesCertificateSecrets lists, by suffix after the release name, the
// Secrets whose certificates a backup checks. Kamaji and cert-manager rotate
// the leaves; the CAs are irreplaceable, so a restore needs them valid.
var kubernetesCertificateSecrets = []string{
"ca",
"front-proxy-ca-certificate",
"api-server-certificate",
"api-server-kubelet-client-certificate",
"front-proxy-client-certificate",
"talos-ca",
"talos-tls-cert",
}
// kubernetesUnlabelledSecrets are issued by cert-manager without the Helm
// instance label the Velero selector relies on; the driver adds it before the
// backup so the Talos PKI is not silently left out.
var kubernetesUnlabelledSecrets = []string{"talos-ca", "talos-tls-cert"}
func kubernetesReleaseName(appName string) string {
return kubernetesReleasePrefix + appName
}
func validateKubernetesApplicationRef(ref corev1.TypedLocalObjectReference) error {
if ref.Kind != kubernetesAppKind {
return fmt.Errorf("kubernetes strategy supports applicationRef.kind %s, got %q", kubernetesAppKind, ref.Kind)
}
if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup {
return fmt.Errorf("kubernetes strategy supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
}
return nil
}
func kubernetesObjectKey(namespace, appName, backupName string) string {
return fmt.Sprintf("%s/%s/%s.etcd.json", namespace, appName, backupName)
}
// kubernetesDatastore is where the tenant cluster keeps its state: a key
// prefix in the etcd DataStore shared by every cluster of the tenant.
type kubernetesDatastore struct {
// ControlPlaneUID identifies the cluster across a delete and re-create
// under the same name, which Prefix does not on a Kamaji that keys the
// schema on the namespace and name.
ControlPlaneUID string
Prefix string
Endpoints []string
CertificateSecret string
// SourcePrefix is the prefix the restored dump was taken under. It
// differs from Prefix when a deleted cluster is rebuilt: Kamaji keys a new
// TenantControlPlane's schema on its UID.
SourcePrefix string
}
func (d kubernetesDatastore) templateContext() map[string]any {
source := d.SourcePrefix
if source == "" {
source = d.Prefix
}
return map[string]any{
"Prefix": d.Prefix,
"SourcePrefix": source,
"Endpoints": strings.Join(d.Endpoints, ","),
"CertificateSecret": d.CertificateSecret,
}
}
// etcdEndpointURL gives a scheme to a DataStore endpoint: Kamaji lists them as
// host:port and always talks TLS to an etcd DataStore.
func etcdEndpointURL(endpoint string) string {
if strings.Contains(endpoint, "://") {
return endpoint
}
return "https://" + endpoint
}
// resolveKubernetesDatastore reads the cluster's key prefix and client
// certificate from its TenantControlPlane and the client URLs from the
// DataStore it points at.
func resolveKubernetesDatastore(ctx context.Context, dyn dynamic.Interface, namespace, appName string) (*kubernetesDatastore, error) {
release := kubernetesReleaseName(appName)
tcp, err := dyn.Resource(tenantControlPlaneGVR).Namespace(namespace).Get(ctx, release, metav1.GetOptions{})
if err != nil {
return nil, err
}
if driver, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "driver"); driver != "" && driver != "etcd" {
return nil, fmt.Errorf("TenantControlPlane %s/%s uses the %q datastore driver; only etcd is supported", namespace, release, driver)
}
prefix, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema")
if prefix == "" {
prefix, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStoreSchema")
}
if prefix == "" {
return nil, fmt.Errorf("TenantControlPlane %s/%s reports no datastore schema yet", namespace, release)
}
dsName, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "dataStoreName")
if dsName == "" {
dsName, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStore")
}
if dsName == "" {
return nil, fmt.Errorf("TenantControlPlane %s/%s names no DataStore", namespace, release)
}
certSecret, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "certificate", "secretName")
if certSecret == "" {
certSecret = release + "-datastore-certificate"
}
ds, err := dyn.Resource(kamajiDataStoreGVR).Get(ctx, dsName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("get DataStore %s: %w", dsName, err)
}
raw, _, _ := unstructured.NestedStringSlice(ds.Object, "spec", "endpoints")
if len(raw) == 0 {
return nil, fmt.Errorf("DataStore %s lists no endpoints", dsName)
}
endpoints := make([]string, 0, len(raw))
for _, e := range raw {
endpoints = append(endpoints, etcdEndpointURL(e))
}
return &kubernetesDatastore{ControlPlaneUID: string(tcp.GetUID()), Prefix: prefix, Endpoints: endpoints, CertificateSecret: certSecret}, nil
}
// kubernetesCertificateWarnings reports every certificate in the given Secrets
// that expires inside its window: caWindow for CAs, leafWindow for the rest.
// Output is sorted so a condition message is stable across reconciles.
func kubernetesCertificateWarnings(secrets []corev1.Secret, now time.Time, caWindow, leafWindow time.Duration) []string {
seen := map[string]bool{}
var out []string
for _, s := range secrets {
for key, data := range s.Data {
if !strings.HasSuffix(key, ".crt") {
continue
}
for rest := data; ; {
var block *pem.Block
block, rest = pem.Decode(rest)
if block == nil {
break
}
if block.Type != "CERTIFICATE" {
continue
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
continue
}
window := leafWindow
kind := "certificate"
if cert.IsCA {
window = caWindow
kind = "CA"
}
if !cert.NotAfter.Before(now.Add(window)) {
continue
}
msg := fmt.Sprintf("%s %s/%s (CN=%s) expires %s", kind, s.Name, key, cert.Subject.CommonName, cert.NotAfter.UTC().Format(time.RFC3339))
if !seen[msg] {
seen[msg] = true
out = append(out, msg)
}
}
}
}
sort.Strings(out)
return out
}
// kubernetesHelmReleaseNames returns the HelmRelease of the cluster followed
// by those of its worker pools. Pools live in separate kubernetes-nodes
// releases; the MachineDeployments of the cluster name them, which also
// covers the older layout where the cluster release owned its pools.
func kubernetesHelmReleaseNames(mds []unstructured.Unstructured, release string) []string {
names := []string{release}
for _, md := range mds {
if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster != release {
continue
}
if owner := md.GetAnnotations()[helmReleaseNameAnnotation]; owner != "" && !slices.Contains(names, owner) {
names = append(names, owner)
}
}
return names
}
// sanitizeHelmRelease keeps what is needed to recreate a HelmRelease and
// drops server-set metadata and status.
func sanitizeHelmRelease(hr *unstructured.Unstructured) map[string]any {
labels := map[string]string{}
for k, v := range hr.GetLabels() {
if !strings.HasPrefix(k, "velero.io/") {
labels[k] = v
}
}
meta := map[string]any{
"name": hr.GetName(),
"namespace": hr.GetNamespace(),
}
if len(labels) > 0 {
meta["labels"] = labels
}
spec, _, _ := unstructured.NestedMap(hr.Object, "spec")
delete(spec, "suspend")
return map[string]any{
"apiVersion": hr.GetAPIVersion(),
"kind": hr.GetKind(),
"metadata": meta,
"spec": spec,
}
}
// kubernetesSchemaPattern is every datastore schema Kamaji assigns: a
// TenantControlPlane UID, or <namespace>_<name> on an older Kamaji. The Job
// turns a schema into the prefix it runs `etcdctl del --prefix` on in the
// etcd every cluster of the tenant shares, so a value that is empty, holds a
// slash or a dot, or comes from an edited Backup must never reach it.
var kubernetesSchemaPattern = regexp.MustCompile(`^[a-z0-9_-]+$`)
func validateKubernetesDatastore(ds kubernetesDatastore) error {
if !kubernetesSchemaPattern.MatchString(ds.Prefix) {
return fmt.Errorf("refusing datastore schema %q: want %s", ds.Prefix, kubernetesSchemaPattern)
}
if ds.SourcePrefix != "" && !kubernetesSchemaPattern.MatchString(ds.SourcePrefix) {
return fmt.Errorf("refusing source datastore schema %q: want %s", ds.SourcePrefix, kubernetesSchemaPattern)
}
return nil
}
func renderKubernetesTemplate(
tmpl corev1.PodTemplateSpec,
appName, namespace, mode, objectKey string,
ds kubernetesDatastore,
parameters map[string]string,
) (*corev1.PodTemplateSpec, error) {
if err := validateKubernetesDatastore(ds); err != nil {
return nil, err
}
return template.Template(&tmpl, map[string]any{
"Release": map[string]string{
"Name": appName,
"Namespace": namespace,
"Prefix": kubernetesReleasePrefix,
},
"Mode": mode,
"ObjectKey": objectKey,
"Datastore": ds.templateContext(),
"Parameters": parameters,
})
}
func kubernetesStrategyParameters(b *backupsv1alpha1.Backup) map[string]string {
out := map[string]string{}
for k, v := range b.Spec.DriverMetadata {
if p := strings.TrimPrefix(k, kubernetesParamPrefix); p != k && p != "" {
out[p] = v
}
}
return out
}
func kubernetesBackupDatastore(b *backupsv1alpha1.Backup) kubernetesDatastore {
ds := kubernetesDatastore{
Prefix: b.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey],
CertificateSecret: kubernetesReleaseName(b.Spec.ApplicationRef.Name) + "-datastore-certificate",
}
if e := b.Spec.DriverMetadata[kubernetesDatastoreEndpointsMetaKey]; e != "" {
ds.Endpoints = strings.Split(e, ",")
}
return ds
}
func kubernetesBackupTTL(s *strategyv1alpha1.Kubernetes) time.Duration {
if s.Spec.Velero.Spec.TTL.Duration > 0 {
return s.Spec.Velero.Spec.TTL.Duration
}
return kubernetesDefaultBackupTTL
}
func kubernetesCertThreshold(s *strategyv1alpha1.Kubernetes) time.Duration {
if s.Spec.CertificateWarningThreshold != nil && s.Spec.CertificateWarningThreshold.Duration > 0 {
return s.Spec.CertificateWarningThreshold.Duration
}
return kubernetesDefaultCertThreshold
}
// ---------------------------------------------------------------------------
// BackupJob path
// ---------------------------------------------------------------------------
// reconcileKubernetes drives a BackupJob for a tenant Kubernetes application in
// three steps, each observed on later reconciles: check the control-plane
// certificates, dump the cluster's etcd prefix with a Job, then back up the
// host-side Secrets and volumes with Velero. The Backup records what a restore
// needs to put both halves back.
func (r *BackupJobReconciler) reconcileKubernetes(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) {
logger := getLogger(ctx)
logger.Debug("reconciling Kubernetes strategy", "backupjob", j.Name, "phase", j.Status.Phase)
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded ||
j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
return ctrl.Result{}, nil
}
if err := validateKubernetesApplicationRef(j.Spec.ApplicationRef); err != nil {
return r.markBackupJobFailed(ctx, j, err.Error())
}
if j.Status.StartedAt == nil {
fresh := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil {
return ctrl.Result{}, err
}
if fresh.Status.StartedAt == nil {
base := fresh.DeepCopy()
now := metav1.Now()
fresh.Status.StartedAt = &now
fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
j.Status.StartedAt = fresh.Status.StartedAt
j.Status.Phase = fresh.Status.Phase
}
strategy := &strategyv1alpha1.Kubernetes{}
if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil {
if apierrors.IsNotFound(err) {
return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name)
}
return ctrl.Result{}, err
}
app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef)
if err != nil {
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application not found: %s/%s", j.Namespace, j.Spec.ApplicationRef.Name))
}
return ctrl.Result{}, err
}
appName := j.Spec.ApplicationRef.Name
release := kubernetesReleaseName(appName)
ds, err := resolveKubernetesDatastore(ctx, r.Interface, j.Namespace, appName)
if err != nil {
if apierrors.IsNotFound(err) {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s has no TenantControlPlane: %v", j.Namespace, appName, err))
}
return r.markBackupJobFailed(ctx, j, err.Error())
}
// A restore that failed partway leaves the control plane stopped on a
// possibly partial prefix, marked with the replica count it had. Backing
// that up would turn the damage into a recovery point, so a run that has
// not dumped anything yet refuses.
if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: jobNameForBackupJob(j)}, &batchv1.Job{}); apierrors.IsNotFound(err) {
kcp, kerr := r.Interface.Resource(kamajiControlPlaneGVR).Namespace(j.Namespace).Get(ctx, release, metav1.GetOptions{})
if kerr != nil && !apierrors.IsNotFound(kerr) && !apimeta.IsNoMatchError(kerr) {
return ctrl.Result{}, kerr
}
if kerr == nil {
if _, stopped := kcp.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; stopped {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s is stopped by an unfinished restore; finish or repair the restore before backing it up", j.Namespace, appName))
}
}
} else if err != nil {
return ctrl.Result{}, err
}
if apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates) == nil {
return r.checkKubernetesCertificates(ctx, j, strategy, release)
}
objectKey := kubernetesObjectKey(j.Namespace, appName, j.Name)
rendered, err := renderKubernetesTemplate(strategy.Spec.Template, appName, j.Namespace, kubernetesModeBackup, objectKey, *ds, resolved.Parameters)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Kubernetes strategy: %v", err))
}
dump, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, jobNameForBackupJob(j),
map[string]string{
kubernetesLabelMode: kubernetesModeBackup,
backupsv1alpha1.OwningJobNameLabel: j.Name,
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
}, rendered)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure etcd dump Job: %v", err))
}
switch jobConditionState(dump) {
case batchv1.JobFailed:
msg := jobFailureMessage(dump)
if msg == "" {
msg = "etcd dump Job reported Failed"
}
return r.markBackupJobFailed(ctx, j, msg)
case batchv1.JobComplete:
default:
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
veleroBackups := &velerov1.BackupList{}
if err := r.List(ctx, veleroBackups, client.InNamespace(veleroNamespace), client.MatchingLabels{
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
backupsv1alpha1.OwningJobNameLabel: j.Name,
}); err != nil {
return ctrl.Result{}, err
}
if len(veleroBackups.Items) == 0 {
if err := r.labelKubernetesSecrets(ctx, j.Namespace, release); err != nil {
return ctrl.Result{}, err
}
if err := r.createKubernetesVeleroBackup(ctx, j, strategy, app, resolved); err != nil {
return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, fmt.Sprintf("failed to create Velero Backup: %v", err))
}
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
if len(veleroBackups.Items) > 1 {
return r.markBackupJobFailed(ctx, j, "more than one Velero Backup references this BackupJob")
}
vb := &veleroBackups.Items[0]
switch vb.Status.Phase {
case velerov1.BackupPhaseCompleted:
case velerov1.BackupPhaseFailed, velerov1.BackupPhasePartiallyFailed, velerov1.BackupPhaseFailedValidation:
return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, vb))
default:
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
if j.Status.BackupRef != nil {
return ctrl.Result{}, nil
}
// A failed create is retried rather than failing the run: the dump and
// the Velero backup already exist, and no Backup would own them.
artifact, err := r.createKubernetesBackupArtifact(ctx, j, resolved, vb, *ds, objectKey)
if err != nil {
return ctrl.Result{}, fmt.Errorf("create Backup artifact: %w", err)
}
now := metav1.Now()
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
j.Status.CompletedAt = &now
j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionTrue,
Reason: "BackupCompleted",
Message: "etcd prefix and Velero backup completed",
})
return ctrl.Result{}, r.Status().Update(ctx, j)
}
// failKubernetesBackupAfterDump fails a run whose etcd dump is already in the
// bucket. No Backup will own the dump, so a cleanup-mode Job owned by the
// BackupJob removes it; the dump holds every Secret of the cluster.
func (r *BackupJobReconciler) failKubernetesBackupAfterDump(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, ds kubernetesDatastore, objectKey string, resolved *ResolvedBackupConfig, msg string) (ctrl.Result, error) {
rendered, err := renderKubernetesTemplate(strategy.Spec.Template, j.Spec.ApplicationRef.Name, j.Namespace, kubernetesModeCleanup, objectKey, ds, resolved.Parameters)
if err == nil {
_, err = ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, j.Name+"-discard",
map[string]string{
kubernetesLabelMode: kubernetesModeCleanup,
backupsv1alpha1.OwningJobNameLabel: j.Name,
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
}, rendered)
}
if err != nil {
msg = fmt.Sprintf("%s; the etcd dump %s could not be scheduled for deletion: %v", msg, objectKey, err)
}
return r.markBackupJobFailed(ctx, j, msg)
}
// checkKubernetesCertificates records on the BackupJob whether any
// control-plane certificate expires inside its window. It never fails the
// run: an expiring certificate is a reason to act, not to skip the backup.
func (r *BackupJobReconciler) checkKubernetesCertificates(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, release string) (ctrl.Result, error) {
var secrets []corev1.Secret
for _, suffix := range kubernetesCertificateSecrets {
s := corev1.Secret{}
err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: release + "-" + suffix}, &s)
if apierrors.IsNotFound(err) {
continue
}
if err != nil {
return ctrl.Result{}, err
}
secrets = append(secrets, s)
}
warnings := kubernetesCertificateWarnings(secrets, time.Now(), kubernetesBackupTTL(strategy), kubernetesCertThreshold(strategy))
cond := metav1.Condition{
Type: kubernetesCondCertificates,
Status: metav1.ConditionFalse,
Reason: "CertificatesValid",
Message: "no control-plane certificate expires within the warning window",
}
if len(warnings) > 0 {
cond.Status = metav1.ConditionTrue
cond.Reason = "CertificatesExpiring"
cond.Message = strings.Join(warnings, "; ")
if r.Recorder != nil {
r.Recorder.Event(j, corev1.EventTypeWarning, "CertificatesExpiring", cond.Message)
}
}
apimeta.SetStatusCondition(&j.Status.Conditions, cond)
if err := r.Status().Update(ctx, j); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
func (r *BackupJobReconciler) labelKubernetesSecrets(ctx context.Context, namespace, release string) error {
for _, suffix := range kubernetesUnlabelledSecrets {
s := &corev1.Secret{}
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release + "-" + suffix}, s)
if apierrors.IsNotFound(err) {
continue
}
if err != nil {
return err
}
if s.Labels[helmInstanceLabel] == release {
continue
}
base := s.DeepCopy()
if s.Labels == nil {
s.Labels = map[string]string{}
}
s.Labels[helmInstanceLabel] = release
if err := r.Patch(ctx, s, client.MergeFrom(base)); err != nil {
return fmt.Errorf("label Secret %s/%s: %w", namespace, s.Name, err)
}
}
return nil
}
func (r *BackupJobReconciler) createKubernetesVeleroBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, app map[string]any, resolved *ResolvedBackupConfig) error {
spec, err := template.Template(&strategy.Spec.Velero.Spec, map[string]any{
"Application": app,
"Parameters": resolved.Parameters,
})
if err != nil {
return err
}
// The volumes an in-place restore deletes and replaces are listed before
// Velero starts: each of them is then in the Velero backup unless it was
// deleted first, and a volume created while the backup runs, which Velero
// may have missed, is never deleted by a restore of it.
volumes, err := r.kubernetesDataVolumeNames(ctx, j.Namespace, kubernetesReleaseName(j.Spec.ApplicationRef.Name))
if err != nil {
return err
}
vb := &velerov1.Backup{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s.%s-", j.Namespace, j.Name),
Namespace: veleroNamespace,
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: j.Name,
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
},
Annotations: map[string]string{kubernetesDataVolumesMetaKey: strings.Join(volumes, ",")},
},
Spec: *spec,
}
if err := r.Create(ctx, vb); err != nil {
return err
}
if r.Recorder != nil {
r.Recorder.Event(j, corev1.EventTypeNormal, "VeleroBackupCreated", fmt.Sprintf("Created Velero Backup %s/%s", veleroNamespace, vb.Name))
}
return nil
}
// kubernetesDataVolumeNames lists the claims backing the cluster's
// PersistentVolumes: the DataVolumes and PVCs the CSI driver labels with the
// cluster name. A PVC is listed even without a DataVolume, so an in-place
// restore clears it too rather than leaving Velero to skip it.
func (r *BackupJobReconciler) kubernetesDataVolumeNames(ctx context.Context, ns, release string) ([]string, error) {
selector := clusterNameLabel + "=" + release
seen := map[string]bool{}
dvs, err := r.Interface.Resource(dataVolumeGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: selector})
if err != nil {
return nil, err
}
for _, dv := range dvs.Items {
seen[dv.GetName()] = true
}
pvcs := &corev1.PersistentVolumeClaimList{}
if err := r.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{clusterNameLabel: release}); err != nil {
return nil, err
}
for _, pvc := range pvcs.Items {
seen[pvc.Name] = true
}
names := make([]string, 0, len(seen))
for name := range seen {
names = append(names, name)
}
sort.Strings(names)
return names, nil
}
// createKubernetesBackupArtifact records, besides the Velero backup and the
// etcd object, the cluster's HelmReleases and data DataVolumes. The
// HelmReleases are kept here rather than in Velero because selecting them by
// application label would also select the worker system disks, which the
// lineage webhook labels after the same application.
func (r *BackupJobReconciler) createKubernetesBackupArtifact(
ctx context.Context,
j *backupsv1alpha1.BackupJob,
resolved *ResolvedBackupConfig,
vb *velerov1.Backup,
ds kubernetesDatastore,
objectKey string,
) (*backupsv1alpha1.Backup, error) {
release := kubernetesReleaseName(j.Spec.ApplicationRef.Name)
mds, err := r.Interface.Resource(machineDeploymentGVR).Namespace(j.Namespace).List(ctx, metav1.ListOptions{})
if err != nil && !apierrors.IsNotFound(err) && !apimeta.IsNoMatchError(err) {
return nil, err
}
var mdItems []unstructured.Unstructured
if mds != nil {
mdItems = mds.Items
}
var releases []map[string]any
for _, name := range kubernetesHelmReleaseNames(mdItems, release) {
hr, err := r.Interface.Resource(helmReleaseGVR).Namespace(j.Namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
continue
}
return nil, err
}
releases = append(releases, sanitizeHelmRelease(hr))
}
releasesJSON, err := json.Marshal(releases)
if err != nil {
return nil, err
}
driverMD := map[string]string{
veleroBackupNameMetadataKey: vb.Name,
veleroBackupNamespaceMetadataKey: vb.Namespace,
kubernetesObjectMetaKey: objectKey,
kubernetesDatastorePrefixMetaKey: ds.Prefix,
kubernetesDatastoreEndpointsMetaKey: strings.Join(ds.Endpoints, ","),
kubernetesHelmReleasesMetaKey: string(releasesJSON),
kubernetesDataVolumesMetaKey: vb.Annotations[kubernetesDataVolumesMetaKey],
kubernetesControlPlaneUIDMetaKey: ds.ControlPlaneUID,
}
if c := apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates); c != nil && c.Status == metav1.ConditionTrue {
driverMD[kubernetesCertWarningsMetaKey] = c.Message
}
for k, v := range resolved.Parameters {
driverMD[kubernetesParamPrefix+k] = v
}
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Name: j.Name, Namespace: j.Namespace},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: j.Spec.ApplicationRef,
StrategyRef: resolved.StrategyRef,
TakenAt: metav1.Now(),
DriverMetadata: driverMD,
PlanRef: j.Spec.PlanRef,
},
Status: backupsv1alpha1.BackupStatus{
Phase: backupsv1alpha1.BackupPhaseReady,
Artifact: &backupsv1alpha1.BackupArtifact{URI: fmt.Sprintf("velero://%s/%s", vb.Namespace, vb.Name)},
},
}
if err := r.Create(ctx, backup); err != nil {
if !apierrors.IsAlreadyExists(err) {
return nil, err
}
existing := &backupsv1alpha1.Backup{}
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); err != nil {
return nil, err
}
return existing, nil
}
return backup, nil
}
// ---------------------------------------------------------------------------
// Backup deletion cleanup
// ---------------------------------------------------------------------------
// cleanupKubernetesBackup removes both halves of a deleted Backup: the Velero
// backup through a DeleteBackupRequest, and the etcd dump object through a
// cleanup-mode Job, waiting for the Job before the Backup is released. The
// escape hatches match the Redis driver's.
func (r *BackupReconciler) cleanupKubernetesBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
if err := r.cleanupVeleroBackup(ctx, backup); err != nil {
return ctrl.Result{}, err
}
jobName := backup.Name + "-cleanup"
if backup.Annotations[redisSkipArtifactCleanupAnnotation] == "true" {
existing := &batchv1.Job{}
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, existing); err == nil {
_ = r.deleteRedisCleanupJob(ctx, existing)
}
return ctrl.Result{}, nil
}
objectKey := backup.Spec.DriverMetadata[kubernetesObjectMetaKey]
if objectKey == "" {
return ctrl.Result{}, nil
}
strategy := &strategyv1alpha1.Kubernetes{}
if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil {
if apierrors.IsNotFound(err) {
return r.releaseRedisCleanup(ctx, backup, objectKey, "strategy CR is gone"), nil
}
return ctrl.Result{}, err
}
job := &batchv1.Job{}
err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job)
switch {
case apierrors.IsNotFound(err):
terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace)
if nsErr != nil {
return ctrl.Result{}, nsErr
}
if terminating {
return r.releaseRedisCleanup(ctx, backup, objectKey, "namespace is terminating"), nil
}
if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil {
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot project credentials: %v", perr)), nil
}
rendered, rerr := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, backup.Namespace,
kubernetesModeCleanup, objectKey, kubernetesBackupDatastore(backup), kubernetesStrategyParameters(backup))
if rerr != nil {
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cleanup template render failed: %v", rerr)), nil
}
cleanup := buildRedisCleanupJob(backup.Namespace, jobName, rendered)
cleanup.Labels = map[string]string{kubernetesLabelMode: kubernetesModeCleanup}
delete(cleanup.Spec.Template.Labels, redisLabelMode)
maps.Copy(cleanup.Spec.Template.Labels, cleanup.Labels)
if cerr := r.Create(ctx, cleanup); cerr != nil && !apierrors.IsAlreadyExists(cerr) {
if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) {
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil
}
return ctrl.Result{}, cerr
}
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
case err != nil:
return ctrl.Result{}, err
}
if !job.DeletionTimestamp.IsZero() {
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
switch jobConditionState(job) {
case batchv1.JobComplete:
_ = r.deleteRedisCleanupJob(ctx, job)
return ctrl.Result{}, nil
case batchv1.JobFailed:
attempts, _ := strconv.Atoi(backup.Annotations[kubernetesCleanupAttemptsAnnotation])
attempts++
if attempts >= kubernetesCleanupMaxAttempts {
_ = r.deleteRedisCleanupJob(ctx, job)
return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("the cleanup Job failed %d times: %s", attempts, jobFailureMessage(job))), nil
}
base := backup.DeepCopy()
if backup.Annotations == nil {
backup.Annotations = map[string]string{}
}
backup.Annotations[kubernetesCleanupAttemptsAnnotation] = strconv.Itoa(attempts)
if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil {
return ctrl.Result{}, err
}
_ = r.deleteRedisCleanupJob(ctx, job)
return ctrl.Result{RequeueAfter: time.Duration(attempts) * 30 * time.Second}, nil
default:
return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil
}
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -204,6 +204,8 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return r.reconcileRedisRestore(ctx, restoreJob, backup)
case strategyv1alpha1.KafkaStrategyKind:
return r.reconcileKafkaRestore(ctx, restoreJob, backup)
case strategyv1alpha1.KubernetesStrategyKind:
return r.reconcileKubernetesRestore(ctx, restoreJob, backup)
default:
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind))
}
@@ -382,6 +384,12 @@ func (r *RestoreJobReconciler) cleanupOnDelete(ctx context.Context, restoreJob *
case strategyv1alpha1.VeleroStrategyKind:
r.cleanupVeleroRestore(ctx, restoreJob)
case strategyv1alpha1.KubernetesStrategyKind:
// Its Velero Restores are labelled like the Velero driver's, so the
// same cleanup reaps them; a live cluster it stopped is resumed.
r.cleanupVeleroRestore(ctx, restoreJob)
r.cleanupKubernetesRestore(ctx, restoreJob)
case strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.JobStrategyKind, strategyv1alpha1.AltinityStrategyKind, strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MongoDBStrategyKind, strategyv1alpha1.FoundationDBStrategyKind, strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind:
// Nothing to clean up: these drivers don't materialise namespaced
// artifacts that outlive the RestoreJob. (Etcd: the operator-side
@@ -6,9 +6,9 @@ include ../../../hack/package.mk
# PHONY so a stray file/dir named `image` cannot mark the aggregate up to date
# and skip both builds.
.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client
.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup
image: image-backupstrategy-controller image-redis-backup image-backup-http-client
image: image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup
image-backupstrategy-controller:
docker buildx build -f images/backupstrategy-controller/Dockerfile ../../.. \
@@ -46,5 +46,16 @@ image-backup-http-client: image-redis-backup
yq -i '.backupStrategyController.chBackupClientImage = strenv(IMAGE) | .backupStrategyController.rabbitmqBackupClientImage = strenv(IMAGE)' values.yaml
rm -f images/backup-http-client.json
# Chained after image-backup-http-client: one `yq -i` writer at a time.
image-kubernetes-backup: image-backup-http-client
docker buildx build -f images/kubernetes-backup/Dockerfile images/kubernetes-backup \
$(call image-tags,kubernetes-backup,v$(COZYSTACK_VERSION)) \
$(call cache-args,kubernetes-backup) \
--metadata-file images/kubernetes-backup.json \
$(BUILDX_ARGS)
IMAGE="$(REGISTRY)/kubernetes-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubernetes-backup.json -o json -r)" \
yq -i '.backupStrategyController.kubernetesBackupClientImage = strenv(IMAGE)' values.yaml
rm -f images/kubernetes-backup.json
test:
helm unittest .
@@ -0,0 +1,147 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.4
name: kubernetes.strategy.backups.cozystack.io
spec:
group: strategy.backups.cozystack.io
names:
kind: Kubernetes
listKind: KubernetesList
plural: kubernetes
singular: kubernetes
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: |-
Kubernetes defines a backup strategy for a tenant Kubernetes application.
A tenant cluster keeps its state in two places the platform can reach: the
cluster's key prefix in the tenant's shared etcd DataStore, and host-side
objects (control-plane PKI Secrets, the application HelmReleases, and the
DataVolumes that back the tenant's PersistentVolumes). The strategy captures
the first with a one-shot Job and the second with Velero. The two are taken
one after another, so a backup is crash-consistent at best.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: KubernetesSpec specifies the desired behavior of a Kubernetes
backup strategy.
properties:
certificateWarningThreshold:
description: |-
CertificateWarningThreshold is how close to expiry a control-plane
certificate may be before a backup records a warning. Defaults to 720h.
type: string
template:
description: |-
Template holds a PodTemplateSpec for the Job that dumps the cluster's
key prefix from its etcd DataStore to object storage, writes it back on
restore, and deletes the object on cleanup. The Job runs in the
application namespace. Helm-like Go templates are supported in every
string field and are rendered against:
- `.Release.Name` and `.Release.Namespace` — the application's name and
namespace; `.Release.Prefix` is `kubernetes-`.
- `.Mode` — `"backup"`, `"restore"` or `"cleanup"`.
- `.ObjectKey` — the object-storage key of this backup's dump.
- `.Datastore.Prefix` — the cluster's key prefix, without slashes.
- `.Datastore.Endpoints` — comma-separated etcd client URLs.
- `.Datastore.CertificateSecret` — the Secret holding the cluster's
etcd client certificate (`ca.crt`, `server.crt`, `server.key`).
- `.Parameters` — the resolved BackupClass strategy parameters.
Stored schemaless for the same CRD-size reason as redis_types.go.
x-kubernetes-preserve-unknown-fields: true
velero:
description: |-
Velero holds the Velero Backup and Restore specs for the host-side
objects and volumes. Rendered against `.Application` and `.Parameters`,
like the Velero strategy.
x-kubernetes-preserve-unknown-fields: true
required:
- template
- velero
type: object
status:
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
type: object
type: object
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,14 @@
# kubernetes-backup runs the Kubernetes strategy Pod: it dumps a tenant
# cluster's key prefix from its etcd DataStore with etcdctl, moves the dump to
# S3 with curl --aws-sigv4, and writes it back with etcdctl on restore. etcdctl
# matches the etcd the platform deploys. Everything is installed at build time
# so the Pods never run `apk add` and work on air-gapped clusters.
FROM quay.io/coreos/etcd:v3.6.11@sha256:6ae247c7666ceec554c51ba1f9bc8dd2212f975370dbd65710c9ca0e36ae1fff AS etcd
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
RUN apk add --no-cache curl jq ca-certificates coreutils
COPY --from=etcd /usr/local/bin/etcdctl /usr/local/bin/etcdctl
USER 65532:65532
@@ -78,6 +78,13 @@ spec:
apiGroup: strategy.backups.cozystack.io
kind: Redis
name: cozy-default-redis
- application:
apiGroup: apps.cozystack.io
kind: Kubernetes
strategyRef:
apiGroup: strategy.backups.cozystack.io
kind: Kubernetes
name: cozy-default-kubernetes
- application:
apiGroup: apps.cozystack.io
kind: Kafka
@@ -62,7 +62,25 @@ rules:
# through the dynamic client, so no list/watch)
- apiGroups: ["cdi.kubevirt.io"]
resources: ["datavolumes"]
verbs: ["get", "delete"]
verbs: ["get", "list", "delete"]
# Kubernetes strategy: read where a tenant cluster keeps its etcd prefix, and
# stop and resume its control plane, worker pools and cluster-autoscaler around
# an in-place restore.
- apiGroups: ["kamaji.clastix.io"]
resources: ["tenantcontrolplanes", "datastores"]
verbs: ["get"]
- apiGroups: ["controlplane.cluster.x-k8s.io"]
resources: ["kamajicontrolplanes"]
verbs: ["get", "patch"]
- apiGroups: ["cluster.x-k8s.io"]
resources: ["machinedeployments"]
verbs: ["get", "list", "patch"]
- apiGroups: ["cluster.x-k8s.io"]
resources: ["machines"]
verbs: ["list", "patch"]
- apiGroups: ["apps"]
resources: ["deployments"]
verbs: ["get", "patch"]
# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR
# and deletes old. patch additionally lets the DefaultObjectsGate stamp
# reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to
@@ -0,0 +1,304 @@
{{/*
Two template layers, as in strategy-redis-default.yaml: Helm renders this
file, then the strategy controller renders the resulting CR per BackupJob.
Literal "{{ .Foo }}" for the second layer goes through printf.
Gated on both a Velero BSL and a resolved $bucketName: the Velero half of
the backup goes to the BSL and the etcd dump to the bucket. Also gated on
its CRD being established, for the upgrade of an existing install that
first ships this kind: see strategy-mongodb-default.yaml.
*/}}
{{- $bucketName := include "backupstrategy-controller.bucketName" . -}}
{{- $crdEstablished := include "backupstrategy-controller.crdEstablished" (dict "root" . "crd" "kubernetes.strategy.backups.cozystack.io") -}}
{{- if and .Values.velero.bslEnabled $bucketName $crdEstablished -}}
apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Kubernetes
metadata:
name: cozy-default-kubernetes
annotations:
backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
spec:
restartPolicy: Never
activeDeadlineSeconds: {{ int .Values.backupStrategyController.kubernetesBackupActiveDeadlineSeconds }}
volumes:
{{- /*
The cluster's own etcd client certificate, the one its kube-apiserver
uses. Optional so a missing Secret fails the script with a clear cause
instead of wedging the Pod on FailedMount.
*/}}
- name: etcd-client
secret:
secretName: {{ printf "{{ .Datastore.CertificateSecret }}" | quote }}
optional: true
{{- if .Values.backupStorage.endpointCASecretName }}
- name: s3-ca
secret:
secretName: {{ .Values.backupStorage.endpointCASecretName | quote }}
optional: true
{{- end }}
containers:
- name: kubernetes-backup
image: "{{ .Values.backupStrategyController.kubernetesBackupClientImage }}"
imagePullPolicy: IfNotPresent
volumeMounts:
- name: etcd-client
mountPath: /etc/etcd-client
readOnly: true
{{- if .Values.backupStorage.endpointCASecretName }}
- name: s3-ca
mountPath: /etc/s3-ca
readOnly: true
{{- end }}
env:
- name: ETCDCTL_ENDPOINTS
value: {{ printf "{{ .Datastore.Endpoints }}" | quote }}
- name: S3_ENDPOINT
valueFrom:
secretKeyRef:
name: cozy-backups-creds
key: endpoint
optional: true
- name: S3_BUCKET
valueFrom:
secretKeyRef:
name: cozy-backups-creds
key: bucketName
optional: true
- name: S3_REGION
valueFrom:
secretKeyRef:
name: cozy-backups-creds
key: region
optional: true
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: cozy-backups-creds
key: AWS_ACCESS_KEY_ID
optional: true
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: cozy-backups-creds
key: AWS_SECRET_ACCESS_KEY
optional: true
- name: S3_INSECURE_SKIP_VERIFY
value: {{ .Values.backupStorage.insecureSkipTLSVerify | quote }}
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
: "${S3_ENDPOINT:?missing endpoint in cozy-backups-creds}"
: "${S3_BUCKET:?missing bucketName in cozy-backups-creds}"
: "${S3_REGION:?missing region in cozy-backups-creds}"
: "${AWS_ACCESS_KEY_ID:?missing AWS_ACCESS_KEY_ID in cozy-backups-creds}"
: "${AWS_SECRET_ACCESS_KEY:?missing AWS_SECRET_ACCESS_KEY in cozy-backups-creds}"
MODE="{{ printf "{{ .Mode }}" }}"
KEY="{{ printf "{{ .ObjectKey }}" }}"
SCHEMA="{{ printf "{{ .Datastore.Prefix }}" }}"
SOURCE_SCHEMA="{{ printf "{{ .Datastore.SourcePrefix }}" }}"
case "${S3_ENDPOINT}" in
http://*|https://*) BASE="${S3_ENDPOINT}" ;;
*) BASE="https://${S3_ENDPOINT}" ;;
esac
TLS=""
case "${BASE}" in
https://*)
if [ -f /etc/s3-ca/ca.crt ]; then TLS="--cacert /etc/s3-ca/ca.crt";
elif [ -f /etc/s3-ca/tls.crt ]; then TLS="--cacert /etc/s3-ca/tls.crt";
elif [ "${S3_INSECURE_SKIP_VERIFY:-false}" = "true" ]; then TLS="-k";
fi
;;
esac
s3() {
curl -fsS ${TLS} \
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@"
}
OBJ="${BASE}/${S3_BUCKET}/${KEY}"
{{- /*
Where a restore keeps the prefix it replaces. One per backup, so
cleanup removes it with the dump. A restore writes it only when it
is absent and deletes it on success, so after a failed restore it
holds the prefix from before the first failed attempt and a retry
cannot overwrite it with a partial one.
*/}}
PRE_KEY="${KEY}.pre-restore"
PRE_OBJ="${BASE}/${S3_BUCKET}/${PRE_KEY}"
if [ "${MODE}" = "cleanup" ]; then
for o in "${OBJ}" "${PRE_OBJ}"; do
echo "deleting ${o#"${BASE}/"}"
code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
-X DELETE "${o}")
case "${code}" in
2??|404) echo "deleted (HTTP ${code})" ;;
*) echo "delete failed: HTTP ${code}" >&2; exit 1 ;;
esac
done
exit 0
fi
if [ "${MODE}" = "verify" ]; then
s3 -I "${OBJ}" > /dev/null
echo "s3://${S3_BUCKET}/${KEY} exists"
exit 0
fi
{{- /*
The prefix comes from the TenantControlPlane: Kamaji's schema is the
TenantControlPlane's UID, or <namespace>_<name> for one created by
an older Kamaji. Refuse anything else: an empty or slash-bearing
value would turn the restore's `del --prefix` into a wipe of every
cluster in the shared etcd.
*/}}
for s in "${SCHEMA}" "${SOURCE_SCHEMA}"; do
case "${s}" in
""|*[!a-z0-9_-]*) echo "refusing unexpected datastore prefix '${s}'" >&2; exit 1 ;;
esac
done
PREFIX="/${SCHEMA}/"
{{- /*
The prefix the dump was taken under. A rebuilt cluster has a new
TenantControlPlane and so a new schema; its keys are written under
the new prefix.
*/}}
SOURCE_PREFIX="/${SOURCE_SCHEMA}/"
for f in ca.crt server.crt server.key; do
[ -f "/etc/etcd-client/${f}" ] || { echo "etcd client certificate is missing /etc/etcd-client/${f}" >&2; exit 1; }
done
export ETCDCTL_CACERT=/etc/etcd-client/ca.crt
export ETCDCTL_CERT=/etc/etcd-client/server.crt
export ETCDCTL_KEY=/etc/etcd-client/server.key
if [ "${MODE}" = "backup" ]; then
etcdctl get "${PREFIX}" --prefix -w json > /tmp/dump.json
count=$(jq '.kvs // [] | length' /tmp/dump.json)
if [ "${count}" -eq 0 ]; then
echo "etcd prefix ${PREFIX} is empty" >&2
exit 1
fi
echo "dumped ${count} keys under ${PREFIX}; uploading s3://${S3_BUCKET}/${KEY}"
s3 -X PUT --upload-file /tmp/dump.json "${OBJ}"
elif [ "${MODE}" = "restore" ]; then
{{- /*
Everything that can fail without touching etcd runs before the
prefix is deleted: the download, the check that every key belongs
to this cluster, and a copy of the prefix as it is now, which is
what an operator restores from if a failure comes after the delete.
*/}}
echo "downloading s3://${S3_BUCKET}/${KEY}"
s3 -o /tmp/dump.json "${OBJ}"
jq -e '.kvs | length > 0' /tmp/dump.json >/dev/null || { echo "dump holds no keys" >&2; exit 1; }
jq -e --arg p "${SOURCE_PREFIX}" 'all(.kvs[]; .key | @base64d | startswith($p))' /tmp/dump.json >/dev/null \
|| { echo "dump holds keys outside ${SOURCE_PREFIX}" >&2; exit 1; }
{{- /*
etcdctl put cannot write an empty value, so a dump holding one
must stop here rather than halfway through the write.
*/}}
jq -e 'all(.kvs[]; (.value // "") != "")' /tmp/dump.json >/dev/null \
|| { echo "dump holds a key with an empty value, which etcdctl cannot write" >&2; exit 1; }
jq -r '.kvs[] | [.key, .value] | @tsv' /tmp/dump.json > /tmp/kvs.tsv
{{- /*
Only a definite answer decides about the copy: an existing one is
kept, a missing one is taken, and any other reply (a 5xx, a
timeout) stops the restore while the prefix is still intact,
rather than overwriting the copy an earlier failed restore left.
*/}}
code=$(curl -sS -o /dev/null -w '%{http_code}' -I ${TLS} \
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
"${PRE_OBJ}") || code=000
case "${code}" in
200) echo "keeping s3://${S3_BUCKET}/${PRE_KEY} from an earlier failed restore" ;;
404)
etcdctl get "${PREFIX}" --prefix -w json > /tmp/before.json
echo "saving the current prefix to s3://${S3_BUCKET}/${PRE_KEY}"
s3 -X PUT --upload-file /tmp/before.json "${PRE_OBJ}"
;;
*) echo "cannot tell whether s3://${S3_BUCKET}/${PRE_KEY} exists: HTTP ${code}" >&2; exit 1 ;;
esac
{{- /*
Delete first so keys created after the backup do not survive the
restore. Events and master leases are bound to etcd leases that
did not survive; written back without one they would never expire,
so they are skipped. They are matched as whole top-level resources
under the prefix, not as a path segment anywhere, which would also
drop every object in a namespace named "events".
*/}}
{{- /*
The termination message tells the driver that this Pod got as far
as changing the prefix: only then is the control plane left
stopped after a failure.
*/}}
echo writing > /dev/termination-log
etcdctl del "${PREFIX}" --prefix
n=0
while IFS="$(printf '\t')" read -r k v; do
key=$(printf '%s' "${k}" | base64 -d)
rel="${key#"${SOURCE_PREFIX}"}"
case "${rel}" in events/*|masterleases/*) continue ;; esac
printf '%s' "${v}" | base64 -d > /tmp/value
etcdctl put "${PREFIX}${rel}" < /tmp/value > /dev/null
n=$((n + 1))
done < /tmp/kvs.tsv
echo "restored ${n} keys under ${PREFIX}"
code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \
--aws-sigv4 "aws:amz:${S3_REGION}:s3" \
--user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \
-X DELETE "${PRE_OBJ}") || code=000
case "${code}" in
2??|404) echo "dropped s3://${S3_BUCKET}/${PRE_KEY}" ;;
*) echo "WARNING: could not drop s3://${S3_BUCKET}/${PRE_KEY} (HTTP ${code}); a later restore would keep it as the pre-restore copy, so delete it by hand" >&2 ;;
esac
else
echo "unknown mode: ${MODE}" >&2
exit 1
fi
resources:
{{- toYaml .Values.backupStrategyController.kubernetesBackupResources | nindent 12 }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
seccompProfile:
type: RuntimeDefault
velero:
{{- /*
Secrets (control-plane and Talos PKI, by the Kamaji and Helm instance
labels) and the DataVolumes and PVCs backing the cluster's
PersistentVolumes (the CSI driver stamps the cluster-name label on them;
worker system disks do not carry it). The HelmReleases are recorded by the
driver on the Backup instead: selecting them by application label would
also select worker disks the lineage webhook labels the same way. The
driver sets the restore's resources and selectors per phase.
*/}}
spec:
includedNamespaces:
- {{ printf "{{ .Application.metadata.namespace }}" | quote }}
orLabelSelectors:
- matchLabels:
kamaji.clastix.io/name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }}
- matchLabels:
app.kubernetes.io/instance: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }}
- matchLabels:
cluster.x-k8s.io/cluster-name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }}
includedResources:
- secrets
- persistentvolumeclaims
- datavolumes.cdi.kubevirt.io
includeClusterResources: false
storageLocation: cozy-default
snapshotVolumes: true
snapshotMoveData: true
ttl: 720h0m0s
itemOperationTimeout: 24h0m0s
restoreSpec:
includeClusterResources: false
{{- end -}}
@@ -22,6 +22,7 @@ templates:
- templates/strategy-foundationdb-default.yaml
- templates/strategy-velero-vminstance-default.yaml
- templates/strategy-velero-vmdisk-default.yaml
- templates/strategy-kubernetes-default.yaml
release:
name: t
@@ -88,11 +89,14 @@ tests:
- hasDocuments:
count: 0
template: templates/strategy-velero-vmdisk-default.yaml
- hasDocuments:
count: 0
template: templates/strategy-kubernetes-default.yaml
- hasDocuments:
count: 0
template: templates/velero-bsl.yaml
- it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all eleven routes (FoundationDB intentionally unbound)"
- it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all twelve routes (FoundationDB intentionally unbound)"
asserts:
- hasDocuments:
count: 1
@@ -103,7 +107,18 @@ tests:
template: templates/backupclass-default.yaml
- lengthEqual:
path: spec.strategies
count: 11
count: 12
template: templates/backupclass-default.yaml
- contains:
path: spec.strategies
content:
application:
apiGroup: apps.cozystack.io
kind: Kubernetes
strategyRef:
apiGroup: strategy.backups.cozystack.io
kind: Kubernetes
name: cozy-default-kubernetes
template: templates/backupclass-default.yaml
- contains:
path: spec.strategies
@@ -240,6 +255,18 @@ tests:
- hasDocuments:
count: 1
template: templates/strategy-velero-vmdisk-default.yaml
- hasDocuments:
count: 1
template: templates/strategy-kubernetes-default.yaml
# The driver renders these per BackupJob; Helm must leave them literal.
- equal:
path: spec.template.spec.volumes[0].secret.secretName
value: "{{ .Datastore.CertificateSecret }}"
template: templates/strategy-kubernetes-default.yaml
- equal:
path: spec.velero.spec.orLabelSelectors[2].matchLabels["cluster.x-k8s.io/cluster-name"]
value: "kubernetes-{{ .Application.metadata.name }}"
template: templates/strategy-kubernetes-default.yaml
- hasDocuments:
count: 1
template: templates/velero-bsl.yaml
@@ -347,6 +374,7 @@ tests:
- {hasDocuments: {count: 0}, template: templates/strategy-foundationdb-default.yaml}
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml}
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml}
- {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml}
# The ungated BackupClass keeps rendering, which is what makes the counts
# above an assertion about the CRD gate rather than about a chart-wide
# render failure.
@@ -432,6 +460,7 @@ tests:
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
@@ -449,6 +478,7 @@ tests:
- {hasDocuments: {count: 0}, template: templates/strategy-redis-default.yaml}
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml}
- {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml}
- {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml}
- it: "with every CRD Established and the BucketClaim resolved, every Strategy CR renders against the COSI bucket name"
# The positive half of the case above, through the same live lookups and no
@@ -471,6 +501,7 @@ tests:
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
- {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}}
@@ -489,6 +520,7 @@ tests:
- {hasDocuments: {count: 1}, template: templates/strategy-redis-default.yaml}
- {hasDocuments: {count: 1}, template: templates/strategy-velero-vminstance-default.yaml}
- {hasDocuments: {count: 1}, template: templates/strategy-velero-vmdisk-default.yaml}
- {hasDocuments: {count: 1}, template: templates/strategy-kubernetes-default.yaml}
- it: "velero.bslEnabled=false: Velero strategies and BSL drop out while other strategies keep rendering"
set:
@@ -13,11 +13,11 @@ release:
namespace: cozy-backup-controller
tests:
- it: "ClusterRole can read and delete DataVolumes"
- it: "ClusterRole can read, list and delete DataVolumes"
asserts:
- contains:
path: rules
content:
apiGroups: ["cdi.kubevirt.io"]
resources: ["datavolumes"]
verbs: ["get", "delete"]
verbs: ["get", "list", "delete"]
@@ -37,6 +37,24 @@ backupStrategyController:
cpu: "1"
memory: 1Gi
ephemeral-storage: 4Gi
# kubernetesBackupClientImage is rendered into the Kubernetes strategy Pod,
# which dumps a tenant cluster's etcd prefix with etcdctl and moves it to S3
# with curl. Built from images/kubernetes-backup/; `make image` re-digests it.
kubernetesBackupClientImage: "ghcr.io/cozystack/cozystack/kubernetes-backup:v1.6.0"
# kubernetesBackupActiveDeadlineSeconds bounds one etcd dump/restore Job.
kubernetesBackupActiveDeadlineSeconds: 3600
# kubernetesBackupResources sizes the Kubernetes strategy Pod. The dump is held
# in ephemeral storage while it moves and jq reads it into memory, so a
# cluster with a large etcd prefix needs more of both.
kubernetesBackupResources:
requests:
cpu: 10m
memory: 64Mi
ephemeral-storage: 256Mi
limits:
cpu: "1"
memory: 1Gi
ephemeral-storage: 2Gi
# The Kafka metadata strategy carries no client image here: unlike the Altinity
# and RabbitMQ drivers, which need only curl (plus jq for Altinity), its driver
# runs the Kafka Admin-API CLI (kafka-topics / kafka-configs), which only the