mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
A tenant Kubernetes application had no backup route in cozy-default, and none of the existing strategies fits it. Its API state is a key prefix in the etcd DataStore the whole tenant shares, so the Etcd strategy's snapshot would roll every tenant cluster back at once; its control-plane CA, service account key and Talos secrets cannot be regenerated without breaking every node and credential issued under them; and its PersistentVolumes are DataVolumes in the tenant namespace that the application's own deletion removes. The new Kubernetes strategy backs up the cluster's etcd prefix with a Job that authenticates with the cluster's own datastore certificate, and the PKI Secrets and data DataVolumes with Velero. The HelmReleases of the cluster and its node pools are recorded on the Backup rather than in Velero, because a label selector for them would also pick up the worker system disks. A backup warns when a CA would expire before the backup does. A restore rebuilds a deleted cluster under its old CA, or rolls a live one back after stopping its control plane and workers, and lets the etcd prefix and the volumes be restored separately. Signed-off-by: Andrey Kolkov <androndo@gmail.com> Assisted-by: LLM
301 lines
14 KiB
Go
301 lines
14 KiB
Go
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
|
|
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
|
)
|
|
|
|
// backupFinalizer guards the strategy-specific side state owned by a Backup.
|
|
// Velero artifacts (the velero.io/Backup CR plus its data on the BSL) are the
|
|
// historical reason for the finalizer, but CNPG-strategy Backups also own
|
|
// side state (a postgresql.cnpg.io/Backup CR) that must be cleaned up here.
|
|
const backupFinalizer = "backups.cozystack.io/cleanup"
|
|
|
|
// legacyVeleroBackupFinalizer is the finalizer name used by previous releases
|
|
// before the cleanup path was generalized across strategies. We still
|
|
// recognise it on existing objects so upgrades complete without leaking
|
|
// finalizers, but new Backups are stamped with backupFinalizer instead.
|
|
const legacyVeleroBackupFinalizer = "backups.cozystack.io/cleanup-velero"
|
|
|
|
// BackupReconciler reconciles Backup objects.
|
|
// It manages a finalizer that ensures strategy-owned side state (Velero or
|
|
// CNPG) is deleted when the cozystack Backup resource is deleted.
|
|
type BackupReconciler struct {
|
|
client.Client
|
|
Scheme *runtime.Scheme
|
|
Recorder record.EventRecorder
|
|
// CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds
|
|
// into the Backup's namespace before spawning its best-effort delete Job.
|
|
CredentialsConfig BackupCredentialsConfig
|
|
}
|
|
|
|
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name)
|
|
|
|
backup := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return ctrl.Result{}, nil
|
|
}
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
// Handle deletion: clean up strategy-specific side state.
|
|
if !backup.DeletionTimestamp.IsZero() {
|
|
if controllerutil.ContainsFinalizer(backup, backupFinalizer) ||
|
|
controllerutil.ContainsFinalizer(backup, legacyVeleroBackupFinalizer) {
|
|
res, err := r.cleanupOnDelete(ctx, backup)
|
|
if err != nil {
|
|
logger.Error(err, "failed to clean up strategy-owned side state")
|
|
return ctrl.Result{}, err
|
|
}
|
|
if res.Requeue || res.RequeueAfter > 0 {
|
|
// Cleanup is still in progress (the Rabbitmq delete Job has not
|
|
// finished); keep the finalizer and requeue so no object is
|
|
// orphaned.
|
|
return res, nil
|
|
}
|
|
|
|
controllerutil.RemoveFinalizer(backup, backupFinalizer)
|
|
controllerutil.RemoveFinalizer(backup, legacyVeleroBackupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("removed finalizer and cleaned up side state", "backup", backup.Name)
|
|
}
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// Ensure finalizer is present
|
|
if !controllerutil.ContainsFinalizer(backup, backupFinalizer) {
|
|
controllerutil.AddFinalizer(backup, backupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("added finalizer to Backup", "backup", backup.Name)
|
|
}
|
|
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// cleanupOnDelete dispatches deletion cleanup to the driver that produced
|
|
// the Backup's side state. Mirrors restorejob_controller's cleanupOnDelete
|
|
// pattern - each strategy knows what it created and how to undo it. Velero
|
|
// owns the velero.io/Backup CR plus archive data on the BSL; CNPG owns the
|
|
// postgresql.cnpg.io/Backup CR; Job owns nothing namespace-scoped.
|
|
// A non-zero ctrl.Result asks the caller to keep the finalizer and requeue -
|
|
// used by the Rabbitmq branch, which waits for its S3 delete Job to finish
|
|
// before letting the Backup be removed, so no object is orphaned.
|
|
func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
kind := strategyKindForBackup(backup)
|
|
logger.V(1).Info("dispatching Backup cleanup", "backup", backup.Name, "strategy", kind)
|
|
switch kind {
|
|
case strategyv1alpha1.CNPGStrategyKind:
|
|
return ctrl.Result{}, r.cleanupCNPGBackup(ctx, backup)
|
|
case strategyv1alpha1.JobStrategyKind:
|
|
// Nothing to clean up: Job-strategy Backups own no namespace-scoped
|
|
// artifacts that survive Backup deletion.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.AltinityStrategyKind:
|
|
// Cozystack Backup deletion does NOT purge the upstream
|
|
// clickhouse-backup archive in object storage. Lifecycle of the
|
|
// archive belongs to the clickhouse-backup tool inside the chi-*
|
|
// Pod (its retention config) or to manual purge through that
|
|
// tool's HTTP API (DELETE /backup/<name>/remote). Surfacing this
|
|
// as an explicit branch (rather than falling through to the
|
|
// Velero default) makes the contract obvious: the Altinity
|
|
// driver does not own the S3 data, so it does not delete it on
|
|
// CR removal. See packages/apps/clickhouse/README.md "Backup
|
|
// orchestration" for tenant-facing guidance.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.MariaDBStrategyKind:
|
|
// Cozystack Backup deletion does NOT delete the operator-side
|
|
// k8s.mariadb.com/Backup CR or the backing S3/PVC archive.
|
|
// Lifecycle of that CR is owned by mariadb-operator's
|
|
// MaxRetention setting and by tenants / explicit teardown flows;
|
|
// the controller's RBAC has no delete verb on
|
|
// k8s.mariadb.com/backups for that reason. Surfacing this as an
|
|
// explicit branch (rather than falling through to the Velero
|
|
// default) keeps the contract honest: the MariaDB driver does
|
|
// not own the archive, so it does not delete it on CR removal.
|
|
// Mirrors the Altinity branch's "we do not own S3" stance.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.MongoDBStrategyKind:
|
|
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
|
|
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
|
|
// operator's — that branch is a no-op, the same "we do not own the
|
|
// archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup
|
|
// lands in the shared cozy-backups bucket with no psmdb task retention to
|
|
// prune it, so the driver OWNS it: this deletes the operator
|
|
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
|
|
// to free the object before the Backup is removed (like Redis / Rabbitmq).
|
|
return r.cleanupMongoDBBackup(ctx, backup)
|
|
case strategyv1alpha1.FoundationDBStrategyKind:
|
|
// Cozystack Backup deletion does NOT delete the operator-side
|
|
// foundationdb.org/FoundationDBBackup CR. That CR drives a
|
|
// continuously streaming backup_agent Deployment; its lifecycle
|
|
// is governed by the next BackupJob's stopOtherFoundationDBBackups
|
|
// call, by manual operator action, or by
|
|
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
|
|
// the archive" contract as Altinity / MariaDB — the explicit
|
|
// branch guards the seam against a future driver refactor that
|
|
// might incidentally stamp velero.io/backup-name onto FDB
|
|
// driverMetadata via a shared helper.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.RabbitmqStrategyKind:
|
|
// Unlike the operator-backed drivers above, the Rabbitmq driver
|
|
// uniquely OWNS its artifact - a plain definitions object it wrote,
|
|
// with the exact key on status.artifact.uri - and no engine retention
|
|
// prunes it. Without an explicit delete a retention-pruned Plan would
|
|
// grow the bucket unboundedly, so this branch deletes the object and
|
|
// WAITS for that delete to complete before the Backup is removed, so no
|
|
// object is orphaned (see cleanupRabbitmqBackup).
|
|
return r.cleanupRabbitmqBackup(ctx, backup)
|
|
case strategyv1alpha1.RedisStrategyKind:
|
|
// Like Rabbitmq, the Redis driver OWNS its artifact - redisObjectKey
|
|
// mints one object per BackupJob and no engine retention prunes it - so
|
|
// this branch deletes that object and WAITS for the delete before the
|
|
// Backup is removed. Without it a retention-pruned Plan grows the bucket
|
|
// unboundedly and the key is lost with the CR (see cleanupRedisBackup).
|
|
return r.cleanupRedisBackup(ctx, backup)
|
|
case strategyv1alpha1.KafkaStrategyKind:
|
|
// Same property as Rabbitmq: the Kafka driver OWNS its artifact (a plain
|
|
// metadata object it wrote with curl, keyed on status.artifact.uri) and
|
|
// no engine retention prunes it. Without an explicit delete a
|
|
// retention-pruned Plan would grow the bucket unboundedly, so this branch
|
|
// deletes the object and WAITS for that delete before the Backup is
|
|
// removed, so no object is orphaned (see cleanupKafkaBackup).
|
|
return r.cleanupKafkaBackup(ctx, backup)
|
|
case strategyv1alpha1.KubernetesStrategyKind:
|
|
// Owns two artifacts: the Velero backup and the etcd dump object.
|
|
return r.cleanupKubernetesBackup(ctx, backup)
|
|
case strategyv1alpha1.VeleroStrategyKind:
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
default:
|
|
// Unknown or empty strategy. Conservative path: try the Velero
|
|
// cleanup since it is keyed on a metadata field that only Velero
|
|
// sets, so it is a no-op for non-Velero Backups. This preserves
|
|
// backward compatibility with pre-existing objects.
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
}
|
|
}
|
|
|
|
// strategyKindForBackup extracts the strategy kind from a Backup's
|
|
// strategyRef. Returns "" when the field is empty so callers can treat that
|
|
// as "unknown" and fall back to a safe default.
|
|
func strategyKindForBackup(backup *backupsv1alpha1.Backup) string {
|
|
return backup.Spec.StrategyRef.Kind
|
|
}
|
|
|
|
// cleanupVeleroBackup deletes the Velero backup and its data from storage
|
|
// by creating a Velero DeleteBackupRequest. A direct Delete of the
|
|
// backup.velero.io resource only removes the Kubernetes object; Velero's
|
|
// BSL sync will recreate it from the object store. The DeleteBackupRequest
|
|
// tells Velero to also purge the data, preventing resurrection.
|
|
func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey]
|
|
if !ok || veleroBackupName == "" {
|
|
logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey]
|
|
if veleroBackupNamespace == "" {
|
|
veleroBackupNamespace = veleroNamespace
|
|
}
|
|
|
|
// Check if the Velero Backup still exists
|
|
veleroBackup := &velerov1.Backup{}
|
|
err := r.Get(ctx, types.NamespacedName{
|
|
Namespace: veleroBackupNamespace,
|
|
Name: veleroBackupName,
|
|
}, veleroBackup)
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
// Create a DeleteBackupRequest so Velero removes backup data from storage.
|
|
// Without this, BSL sync will recreate the backup.velero.io resource.
|
|
dbr := &velerov1.DeleteBackupRequest{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: veleroBackupName + "-",
|
|
Namespace: veleroBackupNamespace,
|
|
},
|
|
Spec: velerov1.DeleteBackupRequestSpec{
|
|
BackupName: veleroBackupName,
|
|
},
|
|
}
|
|
if err := r.Create(ctx, dbr); err != nil {
|
|
if apierrors.IsAlreadyExists(err) {
|
|
logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
logger.Info("created DeleteBackupRequest for Velero backup",
|
|
"name", veleroBackupName, "namespace", veleroBackupNamespace,
|
|
"deleteRequest", dbr.Name)
|
|
return nil
|
|
}
|
|
|
|
// cleanupCNPGBackup deletes the postgresql.cnpg.io/Backup CR that the CNPG
|
|
// driver created for this Cozystack Backup. It does NOT purge the Barman
|
|
// archive in S3 - CNPG itself does not expose a "delete this archive"
|
|
// surface, and removing only the CR matches the operator's expectation that
|
|
// archive lifecycle is governed by spec.backup.retentionPolicy on the
|
|
// Cluster. Tenants who need point-in-time deletion must do it through the
|
|
// object store directly.
|
|
func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
cnpgBackupName, ok := backup.Spec.DriverMetadata[cnpgBackupNameKey]
|
|
if !ok || cnpgBackupName == "" {
|
|
logger.V(1).Info("no cnpg.io backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
cnpgBackup := &cnpgtypes.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: backup.Namespace,
|
|
Name: cnpgBackupName,
|
|
},
|
|
}
|
|
if err := r.Delete(ctx, cnpgBackup); err != nil && !apierrors.IsNotFound(err) {
|
|
return fmt.Errorf("failed to delete cnpg.io/Backup %s/%s: %w", backup.Namespace, cnpgBackupName, err)
|
|
}
|
|
logger.Info("deleted cnpg.io/Backup (or already absent)",
|
|
"name", cnpgBackupName, "namespace", backup.Namespace)
|
|
return nil
|
|
}
|
|
|
|
// SetupWithManager registers the BackupReconciler with the Manager.
|
|
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|
return ctrl.NewControllerManagedBy(mgr).
|
|
For(&backupsv1alpha1.Backup{}).
|
|
Complete(r)
|
|
}
|