mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
A MariaDB restore now checks on the API server that its Backup is not being deleted before it starts, so a restore that started first exists on the API server before the delete. The cleanup read RestoreJobs from the informer cache, which can lag behind, and could delete the dump under a restore it had not seen yet. Reading them uncached closes that window from both sides. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
316 lines
14 KiB
Go
316 lines
14 KiB
Go
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
|
|
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
|
)
|
|
|
|
// backupFinalizer guards the strategy-specific side state owned by a Backup.
|
|
// Velero artifacts (the velero.io/Backup CR plus its data on the BSL) are the
|
|
// historical reason for the finalizer, but CNPG-strategy Backups also own
|
|
// side state (a postgresql.cnpg.io/Backup CR) that must be cleaned up here.
|
|
const backupFinalizer = "backups.cozystack.io/cleanup"
|
|
|
|
// legacyVeleroBackupFinalizer is the finalizer name used by previous releases
|
|
// before the cleanup path was generalized across strategies. We still
|
|
// recognise it on existing objects so upgrades complete without leaking
|
|
// finalizers, but new Backups are stamped with backupFinalizer instead.
|
|
const legacyVeleroBackupFinalizer = "backups.cozystack.io/cleanup-velero"
|
|
|
|
// BackupReconciler reconciles Backup objects.
|
|
// It manages a finalizer that ensures strategy-owned side state (Velero or
|
|
// CNPG) is deleted when the cozystack Backup resource is deleted.
|
|
type BackupReconciler struct {
|
|
client.Client
|
|
Scheme *runtime.Scheme
|
|
Recorder record.EventRecorder
|
|
// CredentialsConfig locates the platform backup credentials: the Job-based
|
|
// cleanups project them into the Backup's namespace, and the MariaDB
|
|
// cleanup reads them to reach the platform bucket.
|
|
CredentialsConfig BackupCredentialsConfig
|
|
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
|
|
// the RestoreJobs of a Backup through it: a restore that started before
|
|
// the delete may not be in the cache yet. Wired in SetupWithManager.
|
|
APIReader client.Reader
|
|
|
|
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
|
// MariaDB cleanup deletes a dump with; nil means a real minio client.
|
|
// Set by unit tests.
|
|
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
|
newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error)
|
|
}
|
|
|
|
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name)
|
|
|
|
backup := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return ctrl.Result{}, nil
|
|
}
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
// Handle deletion: clean up strategy-specific side state.
|
|
if !backup.DeletionTimestamp.IsZero() {
|
|
if controllerutil.ContainsFinalizer(backup, backupFinalizer) ||
|
|
controllerutil.ContainsFinalizer(backup, legacyVeleroBackupFinalizer) {
|
|
res, err := r.cleanupOnDelete(ctx, backup)
|
|
if err != nil {
|
|
logger.Error(err, "failed to clean up strategy-owned side state")
|
|
return ctrl.Result{}, err
|
|
}
|
|
if res.Requeue || res.RequeueAfter > 0 {
|
|
// Cleanup is still in progress (the Rabbitmq delete Job has not
|
|
// finished); keep the finalizer and requeue so no object is
|
|
// orphaned.
|
|
return res, nil
|
|
}
|
|
|
|
controllerutil.RemoveFinalizer(backup, backupFinalizer)
|
|
controllerutil.RemoveFinalizer(backup, legacyVeleroBackupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("removed finalizer and cleaned up side state", "backup", backup.Name)
|
|
}
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// Ensure finalizer is present
|
|
if !controllerutil.ContainsFinalizer(backup, backupFinalizer) {
|
|
controllerutil.AddFinalizer(backup, backupFinalizer)
|
|
if err := r.Update(ctx, backup); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
logger.V(1).Info("added finalizer to Backup", "backup", backup.Name)
|
|
}
|
|
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// cleanupOnDelete dispatches deletion cleanup to the driver that produced
|
|
// the Backup's side state. Mirrors restorejob_controller's cleanupOnDelete
|
|
// pattern - each strategy knows what it created and how to undo it. Velero
|
|
// owns the velero.io/Backup CR plus archive data on the BSL; CNPG owns the
|
|
// postgresql.cnpg.io/Backup CR; Job owns nothing namespace-scoped.
|
|
// A non-zero ctrl.Result asks the caller to keep the finalizer and requeue -
|
|
// used by the Rabbitmq branch, which waits for its S3 delete Job to finish
|
|
// before letting the Backup be removed, so no object is orphaned.
|
|
func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
|
logger := log.FromContext(ctx)
|
|
kind := strategyKindForBackup(backup)
|
|
logger.V(1).Info("dispatching Backup cleanup", "backup", backup.Name, "strategy", kind)
|
|
switch kind {
|
|
case strategyv1alpha1.CNPGStrategyKind:
|
|
return ctrl.Result{}, r.cleanupCNPGBackup(ctx, backup)
|
|
case strategyv1alpha1.JobStrategyKind:
|
|
// Nothing to clean up: Job-strategy Backups own no namespace-scoped
|
|
// artifacts that survive Backup deletion.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.AltinityStrategyKind:
|
|
// Cozystack Backup deletion does NOT purge the upstream
|
|
// clickhouse-backup archive in object storage. Lifecycle of the
|
|
// archive belongs to the clickhouse-backup tool inside the chi-*
|
|
// Pod (its retention config) or to manual purge through that
|
|
// tool's HTTP API (DELETE /backup/<name>/remote). Surfacing this
|
|
// as an explicit branch (rather than falling through to the
|
|
// Velero default) makes the contract obvious: the Altinity
|
|
// driver does not own the S3 data, so it does not delete it on
|
|
// CR removal. See packages/apps/clickhouse/README.md "Backup
|
|
// orchestration" for tenant-facing guidance.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.MariaDBStrategyKind:
|
|
// mariadb-operator's maxRetention prunes only from inside the next
|
|
// backup run of the same prefix and a Plan never deletes a Backup, so
|
|
// a dump on the platform bucket is deleted here or never. A dump in
|
|
// a tenant's own bucket is deleted too: the Backup proves it owns
|
|
// it, and tenants cannot delete Backups, so the delete comes from an
|
|
// administrator or from namespace teardown (see cleanupMariaDBBackup).
|
|
return r.cleanupMariaDBBackup(ctx, backup)
|
|
case strategyv1alpha1.MongoDBStrategyKind:
|
|
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
|
|
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
|
|
// operator's — that branch is a no-op, the same "we do not own the
|
|
// archive" contract as Altinity / FDB. A useSystemBucket backup
|
|
// lands in the shared cozy-backups bucket with no psmdb task retention to
|
|
// prune it, so the driver OWNS it: this deletes the operator
|
|
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
|
|
// to free the object before the Backup is removed (like Redis / Rabbitmq).
|
|
return r.cleanupMongoDBBackup(ctx, backup)
|
|
case strategyv1alpha1.FoundationDBStrategyKind:
|
|
// Cozystack Backup deletion does NOT delete the operator-side
|
|
// foundationdb.org/FoundationDBBackup CR. That CR drives a
|
|
// continuously streaming backup_agent Deployment; its lifecycle
|
|
// is governed by the next BackupJob's stopOtherFoundationDBBackups
|
|
// call, by manual operator action, or by
|
|
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
|
|
// the archive" contract as Altinity — the explicit
|
|
// branch guards the seam against a future driver refactor that
|
|
// might incidentally stamp velero.io/backup-name onto FDB
|
|
// driverMetadata via a shared helper.
|
|
return ctrl.Result{}, nil
|
|
case strategyv1alpha1.RabbitmqStrategyKind:
|
|
// Unlike the operator-backed drivers above, the Rabbitmq driver
|
|
// uniquely OWNS its artifact - a plain definitions object it wrote,
|
|
// with the exact key on status.artifact.uri - and no engine retention
|
|
// prunes it. Without an explicit delete a retention-pruned Plan would
|
|
// grow the bucket unboundedly, so this branch deletes the object and
|
|
// WAITS for that delete to complete before the Backup is removed, so no
|
|
// object is orphaned (see cleanupRabbitmqBackup).
|
|
return r.cleanupRabbitmqBackup(ctx, backup)
|
|
case strategyv1alpha1.RedisStrategyKind:
|
|
// Like Rabbitmq, the Redis driver OWNS its artifact - redisObjectKey
|
|
// mints one object per BackupJob and no engine retention prunes it - so
|
|
// this branch deletes that object and WAITS for the delete before the
|
|
// Backup is removed. Without it a retention-pruned Plan grows the bucket
|
|
// unboundedly and the key is lost with the CR (see cleanupRedisBackup).
|
|
return r.cleanupRedisBackup(ctx, backup)
|
|
case strategyv1alpha1.KafkaStrategyKind:
|
|
// Same property as Rabbitmq: the Kafka driver OWNS its artifact (a plain
|
|
// metadata object it wrote with curl, keyed on status.artifact.uri) and
|
|
// no engine retention prunes it. Without an explicit delete a
|
|
// retention-pruned Plan would grow the bucket unboundedly, so this branch
|
|
// deletes the object and WAITS for that delete before the Backup is
|
|
// removed, so no object is orphaned (see cleanupKafkaBackup).
|
|
return r.cleanupKafkaBackup(ctx, backup)
|
|
case strategyv1alpha1.VeleroStrategyKind:
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
default:
|
|
// Unknown or empty strategy. Conservative path: try the Velero
|
|
// cleanup since it is keyed on a metadata field that only Velero
|
|
// sets, so it is a no-op for non-Velero Backups. This preserves
|
|
// backward compatibility with pre-existing objects.
|
|
return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup)
|
|
}
|
|
}
|
|
|
|
// strategyKindForBackup extracts the strategy kind from a Backup's
|
|
// strategyRef. Returns "" when the field is empty so callers can treat that
|
|
// as "unknown" and fall back to a safe default.
|
|
func strategyKindForBackup(backup *backupsv1alpha1.Backup) string {
|
|
return backup.Spec.StrategyRef.Kind
|
|
}
|
|
|
|
// cleanupVeleroBackup deletes the Velero backup and its data from storage
|
|
// by creating a Velero DeleteBackupRequest. A direct Delete of the
|
|
// backup.velero.io resource only removes the Kubernetes object; Velero's
|
|
// BSL sync will recreate it from the object store. The DeleteBackupRequest
|
|
// tells Velero to also purge the data, preventing resurrection.
|
|
func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey]
|
|
if !ok || veleroBackupName == "" {
|
|
logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey]
|
|
if veleroBackupNamespace == "" {
|
|
veleroBackupNamespace = veleroNamespace
|
|
}
|
|
|
|
// Check if the Velero Backup still exists
|
|
veleroBackup := &velerov1.Backup{}
|
|
err := r.Get(ctx, types.NamespacedName{
|
|
Namespace: veleroBackupNamespace,
|
|
Name: veleroBackupName,
|
|
}, veleroBackup)
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
// Create a DeleteBackupRequest so Velero removes backup data from storage.
|
|
// Without this, BSL sync will recreate the backup.velero.io resource.
|
|
dbr := &velerov1.DeleteBackupRequest{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: veleroBackupName + "-",
|
|
Namespace: veleroBackupNamespace,
|
|
},
|
|
Spec: velerov1.DeleteBackupRequestSpec{
|
|
BackupName: veleroBackupName,
|
|
},
|
|
}
|
|
if err := r.Create(ctx, dbr); err != nil {
|
|
if apierrors.IsAlreadyExists(err) {
|
|
logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err)
|
|
}
|
|
|
|
logger.Info("created DeleteBackupRequest for Velero backup",
|
|
"name", veleroBackupName, "namespace", veleroBackupNamespace,
|
|
"deleteRequest", dbr.Name)
|
|
return nil
|
|
}
|
|
|
|
// cleanupCNPGBackup deletes the postgresql.cnpg.io/Backup CR that the CNPG
|
|
// driver created for this Cozystack Backup. It does NOT purge the Barman
|
|
// archive in S3 - CNPG itself does not expose a "delete this archive"
|
|
// surface, and removing only the CR matches the operator's expectation that
|
|
// archive lifecycle is governed by spec.backup.retentionPolicy on the
|
|
// Cluster. Tenants who need point-in-time deletion must do it through the
|
|
// object store directly.
|
|
func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error {
|
|
logger := log.FromContext(ctx)
|
|
|
|
cnpgBackupName, ok := backup.Spec.DriverMetadata[cnpgBackupNameKey]
|
|
if !ok || cnpgBackupName == "" {
|
|
logger.V(1).Info("no cnpg.io backup name in driverMetadata, nothing to clean up")
|
|
return nil
|
|
}
|
|
|
|
cnpgBackup := &cnpgtypes.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: backup.Namespace,
|
|
Name: cnpgBackupName,
|
|
},
|
|
}
|
|
if err := r.Delete(ctx, cnpgBackup); err != nil && !apierrors.IsNotFound(err) {
|
|
return fmt.Errorf("failed to delete cnpg.io/Backup %s/%s: %w", backup.Namespace, cnpgBackupName, err)
|
|
}
|
|
logger.Info("deleted cnpg.io/Backup (or already absent)",
|
|
"name", cnpgBackupName, "namespace", backup.Namespace)
|
|
return nil
|
|
}
|
|
|
|
// SetupWithManager registers the BackupReconciler with the Manager.
|
|
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|
r.APIReader = mgr.GetAPIReader()
|
|
return ctrl.NewControllerManagedBy(mgr).
|
|
For(&backupsv1alpha1.Backup{}).
|
|
Complete(r)
|
|
}
|
|
|
|
// apiReader returns the uncached reader. Reconcilers built by hand in unit
|
|
// tests have none and fall back to their fake Client, which has no cache to
|
|
// bypass.
|
|
func (r *BackupReconciler) apiReader() client.Reader {
|
|
if r.APIReader != nil {
|
|
return r.APIReader
|
|
}
|
|
return r.Client
|
|
}
|