mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
Deleting a MariaDB Backup left its dump in the bucket for good: mariadb-operator's maxRetention prunes only from inside the next run of the same prefix, and a Plan never deletes a Backup, so on the platform bucket nothing else ever removes it. Every run of an application writes under one prefix, so the key deleted has to be proven to be this Backup's. It is the dump the Backup records, or, for a Backup taken before dumps were recorded, the one the existing ownership match attributes to its operator Backup. That match is recorded on the Backup before the delete, so a retry deletes the same key by name instead of matching again against a bucket that no longer holds it. Without proof the Backup is released with a Warning and nothing is deleted, the operator Backup included, since any later attribution starts from it. The operator Backup is deleted only after its dump, and the delete waits while a RestoreJob still restores from the Backup. The delete runs in the controller through the same minio client the size probe uses, so it verifies TLS the way mariadb-operator did when it wrote the dump. A dump on the platform bucket, which the operator Backup names by its bucket and by the projected credentials Secret, is reached through the platform's current endpoint and keys, still trusting a CA the strategy pinned, so a Backup taken before the endpoint moved is still deleted. Platform credentials that cannot be read fall back to the ones the operator Backup names. A failed delete holds the Backup and retries with a Warning naming the skip-artifact-cleanup annotation, except in a terminating namespace, where the Backup is released with a Warning so it does not hold the namespace. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
301 lines
13 KiB
Go
301 lines
13 KiB
Go
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
|
|
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
|
)
|
|
|
|
// backupFinalizer guards the strategy-specific side state owned by a Backup.
|
|
// Velero artifacts (the velero.io/Backup CR plus its data on the BSL) are the
|
|
// historical reason for the finalizer, but CNPG-strategy Backups also own
|
|
// side state (a postgresql.cnpg.io/Backup CR) that must be cleaned up here.
|
|
const backupFinalizer = "backups.cozystack.io/cleanup"
|
|
|
|
// legacyVeleroBackupFinalizer is the finalizer name used by previous releases
|
|
// before the cleanup path was generalized across strategies. We still
|
|
// recognise it on existing objects so upgrades complete without leaking
|
|
// finalizers, but new Backups are stamped with backupFinalizer instead.
|
|
const legacyVeleroBackupFinalizer = "backups.cozystack.io/cleanup-velero"
|
|
|
|
// BackupReconciler reconciles Backup objects.
|
|
// It manages a finalizer that ensures strategy-owned side state (Velero or
|
|
// CNPG) is deleted when the cozystack Backup resource is deleted.
|
|
type BackupReconciler struct {
|
|
client.Client
|
|
Scheme *runtime.Scheme
|
|
Recorder record.EventRecorder
|
|
// CredentialsConfig locates the platform backup credentials: the Job-based
|
|
// cleanups project them into the Backup's namespace, and the MariaDB
|
|
// cleanup reads them to reach the platform bucket.
|
|
CredentialsConfig BackupCredentialsConfig
|
|
|
|
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
|
// MariaDB cleanup deletes a dump with; nil means a real minio client.
|
|
// Set by unit tests.
|
|
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
|
newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error)
|
|
}
|
|
|
|
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name)
|
|
|
|
backup := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return ctrl.Result{}, nil
|
|
}
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
// Handle deletion: clean up strategy-specific side state.
|
|
if !backup.DeletionTimestamp.IsZero() {
|
|
if controllerutil.ContainsFinalizer(backup, backupFinalizer) ||
|
|
controllerutil.ContainsFinalizer(backup, legacyVeleroBackupFinalizer) {
|
|
res, err := r.cleanupOnDelete(ctx, backup)
|
|
if err != nil {
|
|
logger.Error(err, "failed to clean up strategy-owned side state")
|
|
return ctrl.Result{}, err
|
|
}
|
|
if res.Requeue || res.RequeueAfter > 0 {
|
|
// Cleanup is still in progress (the Rabbitmq delete Job has not
|
|
// finished); keep the finalizer and requeue so no object is
|
|
// orphaned.
|
|
return res, nil
|
|
}
|
|
|
|
controllerutil.RemoveFinalizer(backup, backupFinalizer)
|
|
controllerutil.RemoveFinalizer(backup, legacyVeleroBackupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("removed finalizer and cleaned up side state", "backup", backup.Name)
|
|
}
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// Ensure finalizer is present
|
|
if !controllerutil.ContainsFinalizer(backup, backupFinalizer) {
|
|
controllerutil.AddFinalizer(backup, backupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("added finalizer to Backup", "backup", backup.Name)
|
|
}
|
|
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// cleanupOnDelete dispatches deletion cleanup to the driver that produced
|
|
// the Backup's side state. Mirrors restorejob_controller's cleanupOnDelete
|
|
// pattern - each strategy knows what it created and how to undo it. Velero
|
|
// owns the velero.io/Backup CR plus archive data on the BSL; CNPG owns the
|
|
// postgresql.cnpg.io/Backup CR; Job owns nothing namespace-scoped.
|
|
// A non-zero ctrl.Result asks the caller to keep the finalizer and requeue -
|
|
// used by the Rabbitmq branch, which waits for its S3 delete Job to finish
|
|
// before letting the Backup be removed, so no object is orphaned.
|
|
func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
kind := strategyKindForBackup(backup)
|
|
logger.V(1).Info("dispatching Backup cleanup", "backup", backup.Name, "strategy", kind)
|
|
switch kind {
|
|
case strategyv1alpha1.CNPGStrategyKind:
|
|
return ctrl.Result{}, r.cleanupCNPGBackup(ctx, backup)
|
|
case strategyv1alpha1.JobStrategyKind:
|
|
// Nothing to clean up: Job-strategy Backups own no namespace-scoped
|
|
// artifacts that survive Backup deletion.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.AltinityStrategyKind:
|
|
// Cozystack Backup deletion does NOT purge the upstream
|
|
// clickhouse-backup archive in object storage. Lifecycle of the
|
|
// archive belongs to the clickhouse-backup tool inside the chi-*
|
|
// Pod (its retention config) or to manual purge through that
|
|
// tool's HTTP API (DELETE /backup/<name>/remote). Surfacing this
|
|
// as an explicit branch (rather than falling through to the
|
|
// Velero default) makes the contract obvious: the Altinity
|
|
// driver does not own the S3 data, so it does not delete it on
|
|
// CR removal. See packages/apps/clickhouse/README.md "Backup
|
|
// orchestration" for tenant-facing guidance.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.MariaDBStrategyKind:
|
|
// mariadb-operator's maxRetention prunes only from inside the next
|
|
// backup run of the same prefix and a Plan never deletes a Backup, so
|
|
// a dump on the platform bucket is deleted here or never. A dump in
|
|
// a tenant's own bucket is deleted too: the Backup proves it owns
|
|
// it, and the tenant deleting the Backup is asking for exactly that
|
|
// (see cleanupMariaDBBackup).
|
|
return r.cleanupMariaDBBackup(ctx, backup)
|
|
case strategyv1alpha1.MongoDBStrategyKind:
|
|
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
|
|
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
|
|
// operator's — that branch is a no-op, the same "we do not own the
|
|
// archive" contract as Altinity / FDB. A useSystemBucket backup
|
|
// lands in the shared cozy-backups bucket with no psmdb task retention to
|
|
// prune it, so the driver OWNS it: this deletes the operator
|
|
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
|
|
// to free the object before the Backup is removed (like Redis / Rabbitmq).
|
|
return r.cleanupMongoDBBackup(ctx, backup)
|
|
case strategyv1alpha1.FoundationDBStrategyKind:
|
|
// Cozystack Backup deletion does NOT delete the operator-side
|
|
// foundationdb.org/FoundationDBBackup CR. That CR drives a
|
|
// continuously streaming backup_agent Deployment; its lifecycle
|
|
// is governed by the next BackupJob's stopOtherFoundationDBBackups
|
|
// call, by manual operator action, or by
|
|
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
|
|
// the archive" contract as Altinity — the explicit
|
|
// branch guards the seam against a future driver refactor that
|
|
// might incidentally stamp velero.io/backup-name onto FDB
|
|
// driverMetadata via a shared helper.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.RabbitmqStrategyKind:
|
|
// Unlike the operator-backed drivers above, the Rabbitmq driver
|
|
// uniquely OWNS its artifact - a plain definitions object it wrote,
|
|
// with the exact key on status.artifact.uri - and no engine retention
|
|
// prunes it. Without an explicit delete a retention-pruned Plan would
|
|
// grow the bucket unboundedly, so this branch deletes the object and
|
|
// WAITS for that delete to complete before the Backup is removed, so no
|
|
// object is orphaned (see cleanupRabbitmqBackup).
|
|
return r.cleanupRabbitmqBackup(ctx, backup)
|
|
case strategyv1alpha1.RedisStrategyKind:
|
|
// Like Rabbitmq, the Redis driver OWNS its artifact - redisObjectKey
|
|
// mints one object per BackupJob and no engine retention prunes it - so
|
|
// this branch deletes that object and WAITS for the delete before the
|
|
// Backup is removed. Without it a retention-pruned Plan grows the bucket
|
|
// unboundedly and the key is lost with the CR (see cleanupRedisBackup).
|
|
return r.cleanupRedisBackup(ctx, backup)
|
|
case strategyv1alpha1.KafkaStrategyKind:
|
|
// Same property as Rabbitmq: the Kafka driver OWNS its artifact (a plain
|
|
// metadata object it wrote with curl, keyed on status.artifact.uri) and
|
|
// no engine retention prunes it. Without an explicit delete a
|
|
// retention-pruned Plan would grow the bucket unboundedly, so this branch
|
|
// deletes the object and WAITS for that delete before the Backup is
|
|
// removed, so no object is orphaned (see cleanupKafkaBackup).
|
|
return r.cleanupKafkaBackup(ctx, backup)
|
|
case strategyv1alpha1.VeleroStrategyKind:
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
default:
|
|
// Unknown or empty strategy. Conservative path: try the Velero
|
|
// cleanup since it is keyed on a metadata field that only Velero
|
|
// sets, so it is a no-op for non-Velero Backups. This preserves
|
|
// backward compatibility with pre-existing objects.
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
}
|
|
}
|
|
|
|
// strategyKindForBackup extracts the strategy kind from a Backup's
|
|
// strategyRef. Returns "" when the field is empty so callers can treat that
|
|
// as "unknown" and fall back to a safe default.
|
|
func strategyKindForBackup(backup *backupsv1alpha1.Backup) string {
|
|
return backup.Spec.StrategyRef.Kind
|
|
}
|
|
|
|
// cleanupVeleroBackup deletes the Velero backup and its data from storage
|
|
// by creating a Velero DeleteBackupRequest. A direct Delete of the
|
|
// backup.velero.io resource only removes the Kubernetes object; Velero's
|
|
// BSL sync will recreate it from the object store. The DeleteBackupRequest
|
|
// tells Velero to also purge the data, preventing resurrection.
|
|
func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey]
|
|
if !ok || veleroBackupName == "" {
|
|
logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey]
|
|
if veleroBackupNamespace == "" {
|
|
veleroBackupNamespace = veleroNamespace
|
|
}
|
|
|
|
// Check if the Velero Backup still exists
|
|
veleroBackup := &velerov1.Backup{}
|
|
err := r.Get(ctx, types.NamespacedName{
|
|
Namespace: veleroBackupNamespace,
|
|
Name: veleroBackupName,
|
|
}, veleroBackup)
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
// Create a DeleteBackupRequest so Velero removes backup data from storage.
|
|
// Without this, BSL sync will recreate the backup.velero.io resource.
|
|
dbr := &velerov1.DeleteBackupRequest{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: veleroBackupName + "-",
|
|
Namespace: veleroBackupNamespace,
|
|
},
|
|
Spec: velerov1.DeleteBackupRequestSpec{
|
|
BackupName: veleroBackupName,
|
|
},
|
|
}
|
|
if err := r.Create(ctx, dbr); err != nil {
|
|
if apierrors.IsAlreadyExists(err) {
|
|
logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
logger.Info("created DeleteBackupRequest for Velero backup",
|
|
"name", veleroBackupName, "namespace", veleroBackupNamespace,
|
|
"deleteRequest", dbr.Name)
|
|
return nil
|
|
}
|
|
|
|
// cleanupCNPGBackup deletes the postgresql.cnpg.io/Backup CR that the CNPG
|
|
// driver created for this Cozystack Backup. It does NOT purge the Barman
|
|
// archive in S3 - CNPG itself does not expose a "delete this archive"
|
|
// surface, and removing only the CR matches the operator's expectation that
|
|
// archive lifecycle is governed by spec.backup.retentionPolicy on the
|
|
// Cluster. Tenants who need point-in-time deletion must do it through the
|
|
// object store directly.
|
|
func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
cnpgBackupName, ok := backup.Spec.DriverMetadata[cnpgBackupNameKey]
|
|
if !ok || cnpgBackupName == "" {
|
|
logger.V(1).Info("no cnpg.io backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
cnpgBackup := &cnpgtypes.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: backup.Namespace,
|
|
Name: cnpgBackupName,
|
|
},
|
|
}
|
|
if err := r.Delete(ctx, cnpgBackup); err != nil && !apierrors.IsNotFound(err) {
|
|
return fmt.Errorf("failed to delete cnpg.io/Backup %s/%s: %w", backup.Namespace, cnpgBackupName, err)
|
|
}
|
|
logger.Info("deleted cnpg.io/Backup (or already absent)",
|
|
"name", cnpgBackupName, "namespace", backup.Namespace)
|
|
return nil
|
|
}
|
|
|
|
// SetupWithManager registers the BackupReconciler with the Manager.
|
|
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|
return ctrl.NewControllerManagedBy(mgr).
|
|
For(&backupsv1alpha1.Backup{}).
|
|
Complete(r)
|
|
}
|