mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
Before deleting a dump the cleanup checks every other operator Backup on the same prefix that could have written it, and it read them from the informer cache. A run whose operator Backup had not reached the cache yet but had already uploaded its dump was missed, and its dump deleted as this Backup's. Both the recorded and the legacy paths now list the runs from the API server, as the restore checks already do. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
320 lines
14 KiB
Go
320 lines
14 KiB
Go
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
|
|
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
|
)
|
|
|
|
// backupFinalizer guards the strategy-specific side state owned by a Backup.
|
|
// Velero artifacts (the velero.io/Backup CR plus its data on the BSL) are the
|
|
// historical reason for the finalizer, but CNPG-strategy Backups also own
|
|
// side state (a postgresql.cnpg.io/Backup CR) that must be cleaned up here.
|
|
const backupFinalizer = "backups.cozystack.io/cleanup"
|
|
|
|
// legacyVeleroBackupFinalizer is the finalizer name used by previous releases
|
|
// before the cleanup path was generalized across strategies. We still
|
|
// recognise it on existing objects so upgrades complete without leaking
|
|
// finalizers, but new Backups are stamped with backupFinalizer instead.
|
|
const legacyVeleroBackupFinalizer = "backups.cozystack.io/cleanup-velero"
|
|
|
|
// BackupReconciler reconciles Backup objects.
|
|
// It manages a finalizer that ensures strategy-owned side state (Velero or
|
|
// CNPG) is deleted when the cozystack Backup resource is deleted.
|
|
type BackupReconciler struct {
|
|
client.Client
|
|
Scheme *runtime.Scheme
|
|
Recorder record.EventRecorder
|
|
// CredentialsConfig locates the platform backup credentials: the Job-based
|
|
// cleanups project them into the Backup's namespace, and the MariaDB
|
|
// cleanup reads them to reach the platform bucket.
|
|
CredentialsConfig BackupCredentialsConfig
|
|
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
|
|
// the RestoreJobs and operator Restores of a Backup, and the operator
|
|
// Backups that may own its dump, through it: a restore that started
|
|
// before the delete, or a run that has already uploaded its dump, may
|
|
// not be in the cache yet. Wired in SetupWithManager.
|
|
APIReader client.Reader
|
|
|
|
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
|
// MariaDB cleanup deletes a dump with; nil means a real minio client.
|
|
// Set by unit tests.
|
|
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
|
newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error)
|
|
}
|
|
|
|
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name)
|
|
|
|
backup := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return ctrl.Result{}, nil
|
|
}
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
// Handle deletion: clean up strategy-specific side state.
|
|
if !backup.DeletionTimestamp.IsZero() {
|
|
if controllerutil.ContainsFinalizer(backup, backupFinalizer) ||
|
|
controllerutil.ContainsFinalizer(backup, legacyVeleroBackupFinalizer) {
|
|
res, err := r.cleanupOnDelete(ctx, backup)
|
|
if err != nil {
|
|
logger.Error(err, "failed to clean up strategy-owned side state")
|
|
return ctrl.Result{}, err
|
|
}
|
|
if res.Requeue || res.RequeueAfter > 0 {
|
|
// Cleanup is still in progress (the Rabbitmq delete Job has not
|
|
// finished); keep the finalizer and requeue so no object is
|
|
// orphaned.
|
|
return res, nil
|
|
}
|
|
|
|
controllerutil.RemoveFinalizer(backup, backupFinalizer)
|
|
controllerutil.RemoveFinalizer(backup, legacyVeleroBackupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("removed finalizer and cleaned up side state", "backup", backup.Name)
|
|
}
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// Ensure finalizer is present
|
|
if !controllerutil.ContainsFinalizer(backup, backupFinalizer) {
|
|
controllerutil.AddFinalizer(backup, backupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("added finalizer to Backup", "backup", backup.Name)
|
|
}
|
|
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// cleanupOnDelete dispatches deletion cleanup to the driver that produced
|
|
// the Backup's side state. Mirrors restorejob_controller's cleanupOnDelete
|
|
// pattern - each strategy knows what it created and how to undo it. Velero
|
|
// owns the velero.io/Backup CR plus archive data on the BSL; CNPG owns the
|
|
// postgresql.cnpg.io/Backup CR; Job owns nothing namespace-scoped.
|
|
// A non-zero ctrl.Result asks the caller to keep the finalizer and requeue -
|
|
// used by the Rabbitmq branch, which waits for its S3 delete Job to finish
|
|
// before letting the Backup be removed, so no object is orphaned.
|
|
func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
kind := strategyKindForBackup(backup)
|
|
logger.V(1).Info("dispatching Backup cleanup", "backup", backup.Name, "strategy", kind)
|
|
switch kind {
|
|
case strategyv1alpha1.CNPGStrategyKind:
|
|
return ctrl.Result{}, r.cleanupCNPGBackup(ctx, backup)
|
|
case strategyv1alpha1.JobStrategyKind:
|
|
// Nothing to clean up: Job-strategy Backups own no namespace-scoped
|
|
// artifacts that survive Backup deletion.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.AltinityStrategyKind:
|
|
// Cozystack Backup deletion does NOT purge the upstream
|
|
// clickhouse-backup archive in object storage. Lifecycle of the
|
|
// archive belongs to the clickhouse-backup tool inside the chi-*
|
|
// Pod (its retention config) or to manual purge through that
|
|
// tool's HTTP API (DELETE /backup/<name>/remote). Surfacing this
|
|
// as an explicit branch (rather than falling through to the
|
|
// Velero default) makes the contract obvious: the Altinity
|
|
// driver does not own the S3 data, so it does not delete it on
|
|
// CR removal. See packages/apps/clickhouse/README.md "Backup
|
|
// orchestration" for tenant-facing guidance.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.MariaDBStrategyKind:
|
|
// mariadb-operator's maxRetention prunes only from inside the next
|
|
// backup run of the same prefix and a Plan never deletes a Backup, so
|
|
// a dump on the platform bucket is deleted here or never. A dump in
|
|
// a tenant's own bucket is deleted too: the Backup proves it owns
|
|
// it, and tenants cannot delete Backups, so the delete comes from an
|
|
// administrator or from namespace teardown. Teardown can remove the
|
|
// operator Backup first, and the dump is then left behind with a
|
|
// Warning, since its storage is read from it (see cleanupMariaDBBackup).
|
|
return r.cleanupMariaDBBackup(ctx, backup)
|
|
case strategyv1alpha1.MongoDBStrategyKind:
|
|
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
|
|
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
|
|
// operator's — that branch is a no-op, the same "we do not own the
|
|
// archive" contract as Altinity / FDB. A useSystemBucket backup
|
|
// lands in the shared cozy-backups bucket with no psmdb task retention to
|
|
// prune it, so the driver OWNS it: this deletes the operator
|
|
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
|
|
// to free the object before the Backup is removed (like Redis / Rabbitmq).
|
|
return r.cleanupMongoDBBackup(ctx, backup)
|
|
case strategyv1alpha1.FoundationDBStrategyKind:
|
|
// Cozystack Backup deletion does NOT delete the operator-side
|
|
// foundationdb.org/FoundationDBBackup CR. That CR drives a
|
|
// continuously streaming backup_agent Deployment; its lifecycle
|
|
// is governed by the next BackupJob's stopOtherFoundationDBBackups
|
|
// call, by manual operator action, or by
|
|
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
|
|
// the archive" contract as Altinity — the explicit
|
|
// branch guards the seam against a future driver refactor that
|
|
// might incidentally stamp velero.io/backup-name onto FDB
|
|
// driverMetadata via a shared helper.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.RabbitmqStrategyKind:
|
|
// Unlike the operator-backed drivers above, the Rabbitmq driver
|
|
// uniquely OWNS its artifact - a plain definitions object it wrote,
|
|
// with the exact key on status.artifact.uri - and no engine retention
|
|
// prunes it. Without an explicit delete a retention-pruned Plan would
|
|
// grow the bucket unboundedly, so this branch deletes the object and
|
|
// WAITS for that delete to complete before the Backup is removed, so no
|
|
// object is orphaned (see cleanupRabbitmqBackup).
|
|
return r.cleanupRabbitmqBackup(ctx, backup)
|
|
case strategyv1alpha1.RedisStrategyKind:
|
|
// Like Rabbitmq, the Redis driver OWNS its artifact - redisObjectKey
|
|
// mints one object per BackupJob and no engine retention prunes it - so
|
|
// this branch deletes that object and WAITS for the delete before the
|
|
// Backup is removed. Without it a retention-pruned Plan grows the bucket
|
|
// unboundedly and the key is lost with the CR (see cleanupRedisBackup).
|
|
return r.cleanupRedisBackup(ctx, backup)
|
|
case strategyv1alpha1.KafkaStrategyKind:
|
|
// Same property as Rabbitmq: the Kafka driver OWNS its artifact (a plain
|
|
// metadata object it wrote with curl, keyed on status.artifact.uri) and
|
|
// no engine retention prunes it. Without an explicit delete a
|
|
// retention-pruned Plan would grow the bucket unboundedly, so this branch
|
|
// deletes the object and WAITS for that delete before the Backup is
|
|
// removed, so no object is orphaned (see cleanupKafkaBackup).
|
|
return r.cleanupKafkaBackup(ctx, backup)
|
|
case strategyv1alpha1.VeleroStrategyKind:
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
default:
|
|
// Unknown or empty strategy. Conservative path: try the Velero
|
|
// cleanup since it is keyed on a metadata field that only Velero
|
|
// sets, so it is a no-op for non-Velero Backups. This preserves
|
|
// backward compatibility with pre-existing objects.
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
}
|
|
}
|
|
|
|
// strategyKindForBackup extracts the strategy kind from a Backup's
|
|
// strategyRef. Returns "" when the field is empty so callers can treat that
|
|
// as "unknown" and fall back to a safe default.
|
|
func strategyKindForBackup(backup *backupsv1alpha1.Backup) string {
|
|
return backup.Spec.StrategyRef.Kind
|
|
}
|
|
|
|
// cleanupVeleroBackup deletes the Velero backup and its data from storage
|
|
// by creating a Velero DeleteBackupRequest. A direct Delete of the
|
|
// backup.velero.io resource only removes the Kubernetes object; Velero's
|
|
// BSL sync will recreate it from the object store. The DeleteBackupRequest
|
|
// tells Velero to also purge the data, preventing resurrection.
|
|
func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey]
|
|
if !ok || veleroBackupName == "" {
|
|
logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey]
|
|
if veleroBackupNamespace == "" {
|
|
veleroBackupNamespace = veleroNamespace
|
|
}
|
|
|
|
// Check if the Velero Backup still exists
|
|
veleroBackup := &velerov1.Backup{}
|
|
err := r.Get(ctx, types.NamespacedName{
|
|
Namespace: veleroBackupNamespace,
|
|
Name: veleroBackupName,
|
|
}, veleroBackup)
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
// Create a DeleteBackupRequest so Velero removes backup data from storage.
|
|
// Without this, BSL sync will recreate the backup.velero.io resource.
|
|
dbr := &velerov1.DeleteBackupRequest{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: veleroBackupName + "-",
|
|
Namespace: veleroBackupNamespace,
|
|
},
|
|
Spec: velerov1.DeleteBackupRequestSpec{
|
|
BackupName: veleroBackupName,
|
|
},
|
|
}
|
|
if err := r.Create(ctx, dbr); err != nil {
|
|
if apierrors.IsAlreadyExists(err) {
|
|
logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
logger.Info("created DeleteBackupRequest for Velero backup",
|
|
"name", veleroBackupName, "namespace", veleroBackupNamespace,
|
|
"deleteRequest", dbr.Name)
|
|
return nil
|
|
}
|
|
|
|
// cleanupCNPGBackup deletes the postgresql.cnpg.io/Backup CR that the CNPG
|
|
// driver created for this Cozystack Backup. It does NOT purge the Barman
|
|
// archive in S3 - CNPG itself does not expose a "delete this archive"
|
|
// surface, and removing only the CR matches the operator's expectation that
|
|
// archive lifecycle is governed by spec.backup.retentionPolicy on the
|
|
// Cluster. Tenants who need point-in-time deletion must do it through the
|
|
// object store directly.
|
|
func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
cnpgBackupName, ok := backup.Spec.DriverMetadata[cnpgBackupNameKey]
|
|
if !ok || cnpgBackupName == "" {
|
|
logger.V(1).Info("no cnpg.io backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
cnpgBackup := &cnpgtypes.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: backup.Namespace,
|
|
Name: cnpgBackupName,
|
|
},
|
|
}
|
|
if err := r.Delete(ctx, cnpgBackup); err != nil && !apierrors.IsNotFound(err) {
|
|
return fmt.Errorf("failed to delete cnpg.io/Backup %s/%s: %w", backup.Namespace, cnpgBackupName, err)
|
|
}
|
|
logger.Info("deleted cnpg.io/Backup (or already absent)",
|
|
"name", cnpgBackupName, "namespace", backup.Namespace)
|
|
return nil
|
|
}
|
|
|
|
// SetupWithManager registers the BackupReconciler with the Manager.
|
|
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|
r.APIReader = mgr.GetAPIReader()
|
|
return ctrl.NewControllerManagedBy(mgr).
|
|
For(&backupsv1alpha1.Backup{}).
|
|
Complete(r)
|
|
}
|
|
|
|
// apiReader returns the uncached reader. Reconcilers built by hand in unit
|
|
// tests have none and fall back to their fake Client, which has no cache to
|
|
// bypass.
|
|
func (r *BackupReconciler) apiReader() client.Reader {
|
|
if r.APIReader != nil {
|
|
return r.APIReader
|
|
}
|
|
return r.Client
|
|
}
|