Files
cozystack/internal/backupcontroller/backup_controller.go
T
Andrey Kolkov 0f44cf9e25 feat(backups): add Kubernetes backup strategy for tenant clusters
A tenant Kubernetes application had no backup route in cozy-default, and
none of the existing strategies fits it. Its API state is a key prefix in
the etcd DataStore the whole tenant shares, so the Etcd strategy's snapshot
would roll every tenant cluster back at once; its control-plane CA, service
account key and Talos secrets cannot be regenerated without breaking every
node and credential issued under them; and its PersistentVolumes are
DataVolumes in the tenant namespace that the application's own deletion
removes.

The new Kubernetes strategy backs up the cluster's etcd prefix with a Job
that authenticates with the cluster's own datastore certificate, and the PKI
Secrets and data DataVolumes with Velero. The HelmReleases of the cluster
and its node pools are recorded on the Backup rather than in Velero,
because a label selector for them would also pick up the worker system
disks. A backup warns when a CA would expire before the backup does.

A restore rebuilds a deleted cluster under its old CA, or rolls a live one
back after stopping its control plane and workers, and lets the etcd
prefix and the volumes be restored separately.

Signed-off-by: Andrey Kolkov <androndo@gmail.com>
Assisted-by: LLM
2026-11-10 15:28:34 +07:00

301 lines
14 KiB
Go

package backupcontroller
import (
"context"
"fmt"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
// backupFinalizer guards the strategy-specific side state owned by a Backup.
// Velero artifacts (the velero.io/Backup CR plus its data on the BSL) are the
// historical reason for the finalizer, but CNPG-strategy Backups also own
// side state (a postgresql.cnpg.io/Backup CR) that must be cleaned up here.
const backupFinalizer = "backups.cozystack.io/cleanup"
// legacyVeleroBackupFinalizer is the finalizer name used by previous releases
// before the cleanup path was generalized across strategies. We still
// recognise it on existing objects so upgrades complete without leaking
// finalizers, but new Backups are stamped with backupFinalizer instead.
const legacyVeleroBackupFinalizer = "backups.cozystack.io/cleanup-velero"
// BackupReconciler reconciles Backup objects.
// It manages a finalizer that ensures strategy-owned side state (Velero or
// CNPG) is deleted when the cozystack Backup resource is deleted.
type BackupReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
// CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds
// into the Backup's namespace before spawning its best-effort delete Job.
CredentialsConfig BackupCredentialsConfig
}
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name)
backup := &backupsv1alpha1.Backup{}
if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil {
if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
// Handle deletion: clean up strategy-specific side state.
if !backup.DeletionTimestamp.IsZero() {
if controllerutil.ContainsFinalizer(backup, backupFinalizer) ||
controllerutil.ContainsFinalizer(backup, legacyVeleroBackupFinalizer) {
res, err := r.cleanupOnDelete(ctx, backup)
if err != nil {
logger.Error(err, "failed to clean up strategy-owned side state")
return ctrl.Result{}, err
}
if res.Requeue || res.RequeueAfter > 0 {
// Cleanup is still in progress (the Rabbitmq delete Job has not
// finished); keep the finalizer and requeue so no object is
// orphaned.
return res, nil
}
controllerutil.RemoveFinalizer(backup, backupFinalizer)
controllerutil.RemoveFinalizer(backup, legacyVeleroBackupFinalizer)
if err := r.Update(ctx, backup); err != nil {
return ctrl.Result{}, err
}
logger.V(1).Info("removed finalizer and cleaned up side state", "backup", backup.Name)
}
return ctrl.Result{}, nil
}
// Ensure finalizer is present
if !controllerutil.ContainsFinalizer(backup, backupFinalizer) {
controllerutil.AddFinalizer(backup, backupFinalizer)
if err := r.Update(ctx, backup); err != nil {
return ctrl.Result{}, err
}
logger.V(1).Info("added finalizer to Backup", "backup", backup.Name)
}
return ctrl.Result{}, nil
}
// cleanupOnDelete dispatches deletion cleanup to the driver that produced
// the Backup's side state. Mirrors restorejob_controller's cleanupOnDelete
// pattern - each strategy knows what it created and how to undo it. Velero
// owns the velero.io/Backup CR plus archive data on the BSL; CNPG owns the
// postgresql.cnpg.io/Backup CR; Job owns nothing namespace-scoped.
// A non-zero ctrl.Result asks the caller to keep the finalizer and requeue -
// used by the Rabbitmq branch, which waits for its S3 delete Job to finish
// before letting the Backup be removed, so no object is orphaned.
func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
logger := log.FromContext(ctx)
kind := strategyKindForBackup(backup)
logger.V(1).Info("dispatching Backup cleanup", "backup", backup.Name, "strategy", kind)
switch kind {
case strategyv1alpha1.CNPGStrategyKind:
return ctrl.Result{}, r.cleanupCNPGBackup(ctx, backup)
case strategyv1alpha1.JobStrategyKind:
// Nothing to clean up: Job-strategy Backups own no namespace-scoped
// artifacts that survive Backup deletion.
return ctrl.Result{}, nil
case strategyv1alpha1.AltinityStrategyKind:
// Cozystack Backup deletion does NOT purge the upstream
// clickhouse-backup archive in object storage. Lifecycle of the
// archive belongs to the clickhouse-backup tool inside the chi-*
// Pod (its retention config) or to manual purge through that
// tool's HTTP API (DELETE /backup/<name>/remote). Surfacing this
// as an explicit branch (rather than falling through to the
// Velero default) makes the contract obvious: the Altinity
// driver does not own the S3 data, so it does not delete it on
// CR removal. See packages/apps/clickhouse/README.md "Backup
// orchestration" for tenant-facing guidance.
return ctrl.Result{}, nil
case strategyv1alpha1.MariaDBStrategyKind:
// Cozystack Backup deletion does NOT delete the operator-side
// k8s.mariadb.com/Backup CR or the backing S3/PVC archive.
// Lifecycle of that CR is owned by mariadb-operator's
// MaxRetention setting and by tenants / explicit teardown flows;
// the controller's RBAC has no delete verb on
// k8s.mariadb.com/backups for that reason. Surfacing this as an
// explicit branch (rather than falling through to the Velero
// default) keeps the contract honest: the MariaDB driver does
// not own the archive, so it does not delete it on CR removal.
// Mirrors the Altinity branch's "we do not own S3" stance.
return ctrl.Result{}, nil
case strategyv1alpha1.MongoDBStrategyKind:
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
// operator's — that branch is a no-op, the same "we do not own the
// archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup
// lands in the shared cozy-backups bucket with no psmdb task retention to
// prune it, so the driver OWNS it: this deletes the operator
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
// to free the object before the Backup is removed (like Redis / Rabbitmq).
return r.cleanupMongoDBBackup(ctx, backup)
case strategyv1alpha1.FoundationDBStrategyKind:
// Cozystack Backup deletion does NOT delete the operator-side
// foundationdb.org/FoundationDBBackup CR. That CR drives a
// continuously streaming backup_agent Deployment; its lifecycle
// is governed by the next BackupJob's stopOtherFoundationDBBackups
// call, by manual operator action, or by
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
// the archive" contract as Altinity / MariaDB — the explicit
// branch guards the seam against a future driver refactor that
// might incidentally stamp velero.io/backup-name onto FDB
// driverMetadata via a shared helper.
return ctrl.Result{}, nil
case strategyv1alpha1.RabbitmqStrategyKind:
// Unlike the operator-backed drivers above, the Rabbitmq driver
// uniquely OWNS its artifact - a plain definitions object it wrote,
// with the exact key on status.artifact.uri - and no engine retention
// prunes it. Without an explicit delete a retention-pruned Plan would
// grow the bucket unboundedly, so this branch deletes the object and
// WAITS for that delete to complete before the Backup is removed, so no
// object is orphaned (see cleanupRabbitmqBackup).
return r.cleanupRabbitmqBackup(ctx, backup)
case strategyv1alpha1.RedisStrategyKind:
// Like Rabbitmq, the Redis driver OWNS its artifact - redisObjectKey
// mints one object per BackupJob and no engine retention prunes it - so
// this branch deletes that object and WAITS for the delete before the
// Backup is removed. Without it a retention-pruned Plan grows the bucket
// unboundedly and the key is lost with the CR (see cleanupRedisBackup).
return r.cleanupRedisBackup(ctx, backup)
case strategyv1alpha1.KafkaStrategyKind:
// Same property as Rabbitmq: the Kafka driver OWNS its artifact (a plain
// metadata object it wrote with curl, keyed on status.artifact.uri) and
// no engine retention prunes it. Without an explicit delete a
// retention-pruned Plan would grow the bucket unboundedly, so this branch
// deletes the object and WAITS for that delete before the Backup is
// removed, so no object is orphaned (see cleanupKafkaBackup).
return r.cleanupKafkaBackup(ctx, backup)
case strategyv1alpha1.KubernetesStrategyKind:
// Owns two artifacts: the Velero backup and the etcd dump object.
return r.cleanupKubernetesBackup(ctx, backup)
case strategyv1alpha1.VeleroStrategyKind:
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
default:
// Unknown or empty strategy. Conservative path: try the Velero
// cleanup since it is keyed on a metadata field that only Velero
// sets, so it is a no-op for non-Velero Backups. This preserves
// backward compatibility with pre-existing objects.
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
}
}
// strategyKindForBackup extracts the strategy kind from a Backup's
// strategyRef. Returns "" when the field is empty so callers can treat that
// as "unknown" and fall back to a safe default.
func strategyKindForBackup(backup *backupsv1alpha1.Backup) string {
return backup.Spec.StrategyRef.Kind
}
// cleanupVeleroBackup deletes the Velero backup and its data from storage
// by creating a Velero DeleteBackupRequest. A direct Delete of the
// backup.velero.io resource only removes the Kubernetes object; Velero's
// BSL sync will recreate it from the object store. The DeleteBackupRequest
// tells Velero to also purge the data, preventing resurrection.
func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
logger := log.FromContext(ctx)
veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey]
if !ok || veleroBackupName == "" {
logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up")
return nil
}
veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey]
if veleroBackupNamespace == "" {
veleroBackupNamespace = veleroNamespace
}
// Check if the Velero Backup still exists
veleroBackup := &velerov1.Backup{}
err := r.Get(ctx, types.NamespacedName{
Namespace: veleroBackupNamespace,
Name: veleroBackupName,
}, veleroBackup)
if err != nil {
if apierrors.IsNotFound(err) {
logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName)
return nil
}
return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
}
// Create a DeleteBackupRequest so Velero removes backup data from storage.
// Without this, BSL sync will recreate the backup.velero.io resource.
dbr := &velerov1.DeleteBackupRequest{
ObjectMeta: metav1.ObjectMeta{
GenerateName: veleroBackupName + "-",
Namespace: veleroBackupNamespace,
},
Spec: velerov1.DeleteBackupRequestSpec{
BackupName: veleroBackupName,
},
}
if err := r.Create(ctx, dbr); err != nil {
if apierrors.IsAlreadyExists(err) {
logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName)
return nil
}
return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
}
logger.Info("created DeleteBackupRequest for Velero backup",
"name", veleroBackupName, "namespace", veleroBackupNamespace,
"deleteRequest", dbr.Name)
return nil
}
// cleanupCNPGBackup deletes the postgresql.cnpg.io/Backup CR that the CNPG
// driver created for this Cozystack Backup. It does NOT purge the Barman
// archive in S3 - CNPG itself does not expose a "delete this archive"
// surface, and removing only the CR matches the operator's expectation that
// archive lifecycle is governed by spec.backup.retentionPolicy on the
// Cluster. Tenants who need point-in-time deletion must do it through the
// object store directly.
func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
logger := log.FromContext(ctx)
cnpgBackupName, ok := backup.Spec.DriverMetadata[cnpgBackupNameKey]
if !ok || cnpgBackupName == "" {
logger.V(1).Info("no cnpg.io backup name in driverMetadata, nothing to clean up")
return nil
}
cnpgBackup := &cnpgtypes.Backup{
ObjectMeta: metav1.ObjectMeta{
Namespace: backup.Namespace,
Name: cnpgBackupName,
},
}
if err := r.Delete(ctx, cnpgBackup); err != nil && !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to delete cnpg.io/Backup %s/%s: %w", backup.Namespace, cnpgBackupName, err)
}
logger.Info("deleted cnpg.io/Backup (or already absent)",
"name", cnpgBackupName, "namespace", backup.Namespace)
return nil
}
// SetupWithManager registers the BackupReconciler with the Manager.
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&backupsv1alpha1.Backup{}).
Complete(r)
}