package backupcontroller import ( "context" "fmt" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) // backupFinalizer guards the strategy-specific side state owned by a Backup. // Velero artifacts (the velero.io/Backup CR plus its data on the BSL) are the // historical reason for the finalizer, but CNPG-strategy Backups also own // side state (a postgresql.cnpg.io/Backup CR) that must be cleaned up here. const backupFinalizer = "backups.cozystack.io/cleanup" // legacyVeleroBackupFinalizer is the finalizer name used by previous releases // before the cleanup path was generalized across strategies. We still // recognise it on existing objects so upgrades complete without leaking // finalizers, but new Backups are stamped with backupFinalizer instead. const legacyVeleroBackupFinalizer = "backups.cozystack.io/cleanup-velero" // BackupReconciler reconciles Backup objects. // It manages a finalizer that ensures strategy-owned side state (Velero or // CNPG) is deleted when the cozystack Backup resource is deleted. type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds // into the Backup's namespace before spawning its best-effort delete Job. CredentialsConfig BackupCredentialsConfig } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name) backup := &backupsv1alpha1.Backup{} if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil { if apierrors.IsNotFound(err) { return ctrl.Result{}, nil } return ctrl.Result{}, err } // Handle deletion: clean up strategy-specific side state. if !backup.DeletionTimestamp.IsZero() { if controllerutil.ContainsFinalizer(backup, backupFinalizer) || controllerutil.ContainsFinalizer(backup, legacyVeleroBackupFinalizer) { res, err := r.cleanupOnDelete(ctx, backup) if err != nil { logger.Error(err, "failed to clean up strategy-owned side state") return ctrl.Result{}, err } if res.Requeue || res.RequeueAfter > 0 { // Cleanup is still in progress (the Rabbitmq delete Job has not // finished); keep the finalizer and requeue so no object is // orphaned. return res, nil } controllerutil.RemoveFinalizer(backup, backupFinalizer) controllerutil.RemoveFinalizer(backup, legacyVeleroBackupFinalizer) if err := r.Update(ctx, backup); err != nil { return ctrl.Result{}, err } logger.V(1).Info("removed finalizer and cleaned up side state", "backup", backup.Name) } return ctrl.Result{}, nil } // Ensure finalizer is present if !controllerutil.ContainsFinalizer(backup, backupFinalizer) { controllerutil.AddFinalizer(backup, backupFinalizer) if err := r.Update(ctx, backup); err != nil { return ctrl.Result{}, err } logger.V(1).Info("added finalizer to Backup", "backup", backup.Name) } return ctrl.Result{}, nil } // cleanupOnDelete dispatches deletion cleanup to the driver that produced // the Backup's side state. Mirrors restorejob_controller's cleanupOnDelete // pattern - each strategy knows what it created and how to undo it. Velero // owns the velero.io/Backup CR plus archive data on the BSL; CNPG owns the // postgresql.cnpg.io/Backup CR; Job owns nothing namespace-scoped. // A non-zero ctrl.Result asks the caller to keep the finalizer and requeue - // used by the Rabbitmq branch, which waits for its S3 delete Job to finish // before letting the Backup be removed, so no object is orphaned. func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { logger := log.FromContext(ctx) kind := strategyKindForBackup(backup) logger.V(1).Info("dispatching Backup cleanup", "backup", backup.Name, "strategy", kind) switch kind { case strategyv1alpha1.CNPGStrategyKind: return ctrl.Result{}, r.cleanupCNPGBackup(ctx, backup) case strategyv1alpha1.JobStrategyKind: // Nothing to clean up: Job-strategy Backups own no namespace-scoped // artifacts that survive Backup deletion. return ctrl.Result{}, nil case strategyv1alpha1.AltinityStrategyKind: // Cozystack Backup deletion does NOT purge the upstream // clickhouse-backup archive in object storage. Lifecycle of the // archive belongs to the clickhouse-backup tool inside the chi-* // Pod (its retention config) or to manual purge through that // tool's HTTP API (DELETE /backup//remote). Surfacing this // as an explicit branch (rather than falling through to the // Velero default) makes the contract obvious: the Altinity // driver does not own the S3 data, so it does not delete it on // CR removal. See packages/apps/clickhouse/README.md "Backup // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: // Cozystack Backup deletion does NOT delete the operator-side // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. // Lifecycle of that CR is owned by mariadb-operator's // MaxRetention setting and by tenants / explicit teardown flows; // the controller's RBAC has no delete verb on // k8s.mariadb.com/backups for that reason. Surfacing this as an // explicit branch (rather than falling through to the Velero // default) keeps the contract honest: the MariaDB driver does // not own the archive, so it does not delete it on CR removal. // Mirrors the Altinity branch's "we do not own S3" stance. return ctrl.Result{}, nil case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer // to free the object before the Backup is removed (like Redis / Rabbitmq). return r.cleanupMongoDBBackup(ctx, backup) case strategyv1alpha1.FoundationDBStrategyKind: // Cozystack Backup deletion does NOT delete the operator-side // foundationdb.org/FoundationDBBackup CR. That CR drives a // continuously streaming backup_agent Deployment; its lifecycle // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own // the archive" contract as Altinity / MariaDB — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. return ctrl.Result{}, nil case strategyv1alpha1.RabbitmqStrategyKind: // Unlike the operator-backed drivers above, the Rabbitmq driver // uniquely OWNS its artifact - a plain definitions object it wrote, // with the exact key on status.artifact.uri - and no engine retention // prunes it. Without an explicit delete a retention-pruned Plan would // grow the bucket unboundedly, so this branch deletes the object and // WAITS for that delete to complete before the Backup is removed, so no // object is orphaned (see cleanupRabbitmqBackup). return r.cleanupRabbitmqBackup(ctx, backup) case strategyv1alpha1.RedisStrategyKind: // Like Rabbitmq, the Redis driver OWNS its artifact - redisObjectKey // mints one object per BackupJob and no engine retention prunes it - so // this branch deletes that object and WAITS for the delete before the // Backup is removed. Without it a retention-pruned Plan grows the bucket // unboundedly and the key is lost with the CR (see cleanupRedisBackup). return r.cleanupRedisBackup(ctx, backup) case strategyv1alpha1.KafkaStrategyKind: // Same property as Rabbitmq: the Kafka driver OWNS its artifact (a plain // metadata object it wrote with curl, keyed on status.artifact.uri) and // no engine retention prunes it. Without an explicit delete a // retention-pruned Plan would grow the bucket unboundedly, so this branch // deletes the object and WAITS for that delete before the Backup is // removed, so no object is orphaned (see cleanupKafkaBackup). return r.cleanupKafkaBackup(ctx, backup) case strategyv1alpha1.VeleroStrategyKind: return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup) default: // Unknown or empty strategy. Conservative path: try the Velero // cleanup since it is keyed on a metadata field that only Velero // sets, so it is a no-op for non-Velero Backups. This preserves // backward compatibility with pre-existing objects. return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup) } } // strategyKindForBackup extracts the strategy kind from a Backup's // strategyRef. Returns "" when the field is empty so callers can treat that // as "unknown" and fall back to a safe default. func strategyKindForBackup(backup *backupsv1alpha1.Backup) string { return backup.Spec.StrategyRef.Kind } // cleanupVeleroBackup deletes the Velero backup and its data from storage // by creating a Velero DeleteBackupRequest. A direct Delete of the // backup.velero.io resource only removes the Kubernetes object; Velero's // BSL sync will recreate it from the object store. The DeleteBackupRequest // tells Velero to also purge the data, preventing resurrection. func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error { logger := log.FromContext(ctx) veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] if !ok || veleroBackupName == "" { logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up") return nil } veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey] if veleroBackupNamespace == "" { veleroBackupNamespace = veleroNamespace } // Check if the Velero Backup still exists veleroBackup := &velerov1.Backup{} err := r.Get(ctx, types.NamespacedName{ Namespace: veleroBackupNamespace, Name: veleroBackupName, }, veleroBackup) if err != nil { if apierrors.IsNotFound(err) { logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName) return nil } return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err) } // Create a DeleteBackupRequest so Velero removes backup data from storage. // Without this, BSL sync will recreate the backup.velero.io resource. dbr := &velerov1.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ GenerateName: veleroBackupName + "-", Namespace: veleroBackupNamespace, }, Spec: velerov1.DeleteBackupRequestSpec{ BackupName: veleroBackupName, }, } if err := r.Create(ctx, dbr); err != nil { if apierrors.IsAlreadyExists(err) { logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName) return nil } return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err) } logger.Info("created DeleteBackupRequest for Velero backup", "name", veleroBackupName, "namespace", veleroBackupNamespace, "deleteRequest", dbr.Name) return nil } // cleanupCNPGBackup deletes the postgresql.cnpg.io/Backup CR that the CNPG // driver created for this Cozystack Backup. It does NOT purge the Barman // archive in S3 - CNPG itself does not expose a "delete this archive" // surface, and removing only the CR matches the operator's expectation that // archive lifecycle is governed by spec.backup.retentionPolicy on the // Cluster. Tenants who need point-in-time deletion must do it through the // object store directly. func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error { logger := log.FromContext(ctx) cnpgBackupName, ok := backup.Spec.DriverMetadata[cnpgBackupNameKey] if !ok || cnpgBackupName == "" { logger.V(1).Info("no cnpg.io backup name in driverMetadata, nothing to clean up") return nil } cnpgBackup := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: cnpgBackupName, }, } if err := r.Delete(ctx, cnpgBackup); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete cnpg.io/Backup %s/%s: %w", backup.Namespace, cnpgBackupName, err) } logger.Info("deleted cnpg.io/Backup (or already absent)", "name", cnpgBackupName, "namespace", backup.Namespace) return nil } // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) }