mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 04:33:36 +09:00
fix(mongodb): keep the cleanup escape hatch off legacy operator CRs, and surface a stuck finalizer
The skip-artifact-cleanup annotation reaches releaseMongoDBCleanup before the ownership check, and the operator-CR delete sat outside the finalizer gate, so a legacy backup (no delete-backup finalizer) lost its PerconaServerMongoDBBackup record when a tenant set the annotation to release a stuck Backup or applied it across a namespace during teardown. That record is what a psmdb restore-by-name resolves against, and both the RBAC comment and the docs promise it is never touched. Gate every write in the release on the finalizer being present: an unowned (legacy) CR or one that cannot be read at all is left intact, and the Cozystack Backup still releases. Surface a failed finalizer strip. The release must proceed even against an unreachable CR, but the swallowed patch error left the operator CR holding percona.com/delete-backup — pinning its namespace in Terminating, the outcome the branch exists to prevent — while the Event reported the success case. Emit a FinalizerNotStripped Warning naming the object so an operator can find it. Mark the sibling backup fields (schedule, retentionPolicy, destinationPath, endpointURL, insecureSkipTLSVerify, s3AccessKey, s3SecretKey) as ignored under useSystemBucket, so a tenant editing one instead of the flag itself is not misled by a description that describes only the legacy flow; regenerate the schema, README and mongodb-rd. Pair the insecureSkipTLSVerify rendering assertion with a case that flips the platform knob on, so the two together discriminate the rendered value from a hardcoded default. Assisted-By: LLM Signed-off-by: Andrey Kolkov <androndo@gmail.com>
This commit is contained in:
1 parent
d58322e253
commit
cc51e040ae
8 files changed
+141
-50
No files matched your search
@@ -792,28 +792,49 @@ func (r *BackupReconciler) cleanupMongoDBBackup(ctx context.Context, backup *bac
|
||||
return ctrl.Result{RequeueAfter: psmdbPollInterval}, nil
|
||||
}
|
||||
|
||||
// releaseMongoDBCleanup gives up pruning the archive and lets the Cozystack
|
||||
// Backup (and any enclosing namespace) finish deleting: best-effort, it strips
|
||||
// the driver's delete-backup finalizer from the operator CR so the CR no longer
|
||||
// blocks teardown and deletes that CR, then records a Warning naming the object
|
||||
// left in the bucket. Every apiserver step is best-effort — a failed read or
|
||||
// write is swallowed — because the caller reaches here precisely to let a wedged
|
||||
// Backup go, and must never be blocked by the same unreachable CR. Takes the CR
|
||||
// name rather than a fetched object so it works on the annotation path, which
|
||||
// runs before any read. Mirrors the Redis/Rabbitmq release path.
|
||||
// releaseMongoDBCleanup lets the Cozystack Backup (and any enclosing namespace)
|
||||
// finish deleting when the archive cannot be pruned in place. It only ever
|
||||
// touches an operator CR the driver OWNS — one carrying the delete-backup
|
||||
// finalizer: it strips that finalizer so the CR no longer blocks teardown and
|
||||
// deletes it. A legacy CR (no finalizer) or one that cannot be read at all is
|
||||
// left entirely intact, so the never-deleted contract for legacy backups holds
|
||||
// even though the annotation reaches this path before the ownership check, and
|
||||
// an unreadable CR still releases. The finalizer strip is best-effort — the
|
||||
// release must proceed even against an unreachable CR — but a failed strip is
|
||||
// surfaced (log + Event), because the CR then keeps the finalizer and can hold
|
||||
// its namespace in Terminating, and the operator needs to know which one. Takes
|
||||
// the CR name so it works on the annotation path, which runs before any read.
|
||||
func (r *BackupReconciler) releaseMongoDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, sourceBackupName, reason string) (ctrl.Result, error) {
|
||||
logger := getLogger(ctx)
|
||||
live := &psmdbtypes.PerconaServerMongoDBBackup{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: sourceBackupName}, live); err == nil {
|
||||
if controllerutil.ContainsFinalizer(live, psmdbDeleteBackupFinalizer) {
|
||||
base := live.DeepCopy()
|
||||
controllerutil.RemoveFinalizer(live, psmdbDeleteBackupFinalizer)
|
||||
_ = r.Patch(ctx, live, client.MergeFrom(base))
|
||||
}
|
||||
if live.DeletionTimestamp.IsZero() {
|
||||
_ = r.Delete(ctx, live)
|
||||
}
|
||||
err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: sourceBackupName}, live)
|
||||
if err != nil || !controllerutil.ContainsFinalizer(live, psmdbDeleteBackupFinalizer) {
|
||||
// Legacy backup (no delete-backup finalizer) or an unreadable CR: nothing
|
||||
// of the driver's to strip or delete. Leave the operator CR untouched and
|
||||
// release the Cozystack Backup.
|
||||
logger.Debug("releasing MongoDB Backup; no driver-owned archive to prune",
|
||||
"backup", backup.Name, "sourceBackup", sourceBackupName, "reason", reason)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
getLogger(ctx).Info("releasing MongoDB Backup without pruning its object",
|
||||
|
||||
base := live.DeepCopy()
|
||||
controllerutil.RemoveFinalizer(live, psmdbDeleteBackupFinalizer)
|
||||
if perr := r.Patch(ctx, live, client.MergeFrom(base)); perr != nil {
|
||||
// The release still proceeds (an unreachable CR must not wedge the
|
||||
// Backup), but the CR keeps the finalizer and can pin its namespace in
|
||||
// Terminating — surface which object so an operator can find it.
|
||||
logger.Info("released MongoDB Backup but could not strip the delete-backup finalizer; the operator CR may hold its namespace in Terminating",
|
||||
"backup", backup.Name, "sourceBackup", sourceBackupName, "reason", reason, "error", perr)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "FinalizerNotStripped",
|
||||
"released Backup but could not strip the delete-backup finalizer from PerconaServerMongoDBBackup %s (%v); it may keep the namespace in Terminating", sourceBackupName, perr)
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
if live.DeletionTimestamp.IsZero() {
|
||||
_ = r.Delete(ctx, live)
|
||||
}
|
||||
logger.Info("releasing MongoDB Backup, leaving its shared-bucket object for a lifecycle policy",
|
||||
"backup", backup.Name, "sourceBackup", sourceBackupName, "reason", reason)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted",
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
@@ -1507,4 +1508,57 @@ func TestCleanupMongoDBBackup(t *testing.T) {
|
||||
t.Fatalf("NoMatchError must be treated as gone and release, got res=%+v err=%v", res, err)
|
||||
}
|
||||
})
|
||||
|
||||
// The annotation reaches releaseMongoDBCleanup before the ownership check, so
|
||||
// the release must itself refuse to touch a CR the driver does not own — the
|
||||
// legacy operator-side backup record is what a psmdb restore-by-name resolves
|
||||
// against, and the RBAC comment and docs both promise it is never deleted.
|
||||
t.Run("legacy backup with the skip annotation keeps its operator CR", func(t *testing.T) {
|
||||
c := newMongoDBStrategyTestClient(t, opBackup(false), liveNamespace)
|
||||
r := &BackupReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
|
||||
res, err := r.cleanupMongoDBBackup(context.Background(), newBackup(map[string]string{psmdbSkipArtifactCleanupAnnotation: "true"}))
|
||||
if err != nil || res.RequeueAfter != 0 {
|
||||
t.Fatalf("legacy + annotation must release, got res=%+v err=%v", res, err)
|
||||
}
|
||||
got := getOp(t, c)
|
||||
if !got.DeletionTimestamp.IsZero() {
|
||||
t.Errorf("the escape hatch must not delete a legacy (unowned) operator CR")
|
||||
}
|
||||
})
|
||||
|
||||
// When the finalizer strip fails, the CR keeps percona.com/delete-backup and
|
||||
// can pin its namespace in Terminating; the release still proceeds, but it
|
||||
// must say which object is stuck rather than emit the success-case Event.
|
||||
t.Run("a failed finalizer strip surfaces the stuck object", func(t *testing.T) {
|
||||
s := runtime.NewScheme()
|
||||
_ = scheme.AddToScheme(s)
|
||||
_ = backupsv1alpha1.AddToScheme(s)
|
||||
_ = psmdbtypes.AddToScheme(s)
|
||||
c := clientfake.NewClientBuilder().
|
||||
WithScheme(s).
|
||||
WithObjects(opBackup(true)).
|
||||
WithInterceptorFuncs(interceptor.Funcs{
|
||||
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
||||
if _, ok := obj.(*psmdbtypes.PerconaServerMongoDBBackup); ok {
|
||||
return apierrors.NewServiceUnavailable("apiserver down")
|
||||
}
|
||||
return cl.Patch(ctx, obj, patch, opts...)
|
||||
},
|
||||
}).
|
||||
Build()
|
||||
rec := record.NewFakeRecorder(10)
|
||||
r := &BackupReconciler{Client: c, Recorder: rec}
|
||||
res, err := r.cleanupMongoDBBackup(context.Background(), newBackup(map[string]string{psmdbSkipArtifactCleanupAnnotation: "true"}))
|
||||
if err != nil || res.RequeueAfter != 0 {
|
||||
t.Fatalf("release must proceed even when the strip fails, got res=%+v err=%v", res, err)
|
||||
}
|
||||
select {
|
||||
case ev := <-rec.Events:
|
||||
if !strings.Contains(ev, "FinalizerNotStripped") || !strings.Contains(ev, "op-bk") {
|
||||
t.Errorf("expected a FinalizerNotStripped event naming the object, got %q", ev)
|
||||
}
|
||||
default:
|
||||
t.Errorf("expected an event surfacing the stuck finalizer, got none")
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user