fix(mongodb): keep the cleanup escape hatch off legacy operator CRs, and surface a stuck finalizer

The skip-artifact-cleanup annotation reaches releaseMongoDBCleanup before the
ownership check, and the operator-CR delete sat outside the finalizer gate, so
a legacy backup (no delete-backup finalizer) lost its PerconaServerMongoDBBackup
record when a tenant set the annotation to release a stuck Backup or applied it
across a namespace during teardown. That record is what a psmdb restore-by-name
resolves against, and both the RBAC comment and the docs promise it is never
touched. Gate every write in the release on the finalizer being present: an
unowned (legacy) CR or one that cannot be read at all is left intact, and the
Cozystack Backup still releases.

Surface a failed finalizer strip. The release must proceed even against an
unreachable CR, but the swallowed patch error left the operator CR holding
percona.com/delete-backup — pinning its namespace in Terminating, the outcome
the branch exists to prevent — while the Event reported the success case. Emit
a FinalizerNotStripped Warning naming the object so an operator can find it.

Mark the sibling backup fields (schedule, retentionPolicy, destinationPath,
endpointURL, insecureSkipTLSVerify, s3AccessKey, s3SecretKey) as ignored under
useSystemBucket, so a tenant editing one instead of the flag itself is not
misled by a description that describes only the legacy flow; regenerate the
schema, README and mongodb-rd. Pair the insecureSkipTLSVerify rendering
assertion with a case that flips the platform knob on, so the two together
discriminate the rendered value from a hardcoded default.

Assisted-By: LLM
Signed-off-by: Andrey Kolkov <androndo@gmail.com>
This commit is contained in:
Andrey Kolkov committed 2026-09-22 12:03:58 +04:00
1 parent d58322e253
commit cc51e040ae
8 files changed
+141 -50

No files matched your search

@@ -792,28 +792,49 @@ func (r *BackupReconciler) cleanupMongoDBBackup(ctx context.Context, backup *bac
return ctrl.Result{RequeueAfter: psmdbPollInterval}, nil
}
// releaseMongoDBCleanup gives up pruning the archive and lets the Cozystack
// Backup (and any enclosing namespace) finish deleting: best-effort, it strips
// the driver's delete-backup finalizer from the operator CR so the CR no longer
// blocks teardown and deletes that CR, then records a Warning naming the object
// left in the bucket. Every apiserver step is best-effort — a failed read or
// write is swallowed — because the caller reaches here precisely to let a wedged
// Backup go, and must never be blocked by the same unreachable CR. Takes the CR
// name rather than a fetched object so it works on the annotation path, which
// runs before any read. Mirrors the Redis/Rabbitmq release path.
// releaseMongoDBCleanup lets the Cozystack Backup (and any enclosing namespace)
// finish deleting when the archive cannot be pruned in place. It only ever
// touches an operator CR the driver OWNS — one carrying the delete-backup
// finalizer: it strips that finalizer so the CR no longer blocks teardown and
// deletes it. A legacy CR (no finalizer) or one that cannot be read at all is
// left entirely intact, so the never-deleted contract for legacy backups holds
// even though the annotation reaches this path before the ownership check, and
// an unreadable CR still releases. The finalizer strip is best-effort — the
// release must proceed even against an unreachable CR — but a failed strip is
// surfaced (log + Event), because the CR then keeps the finalizer and can hold
// its namespace in Terminating, and the operator needs to know which one. Takes
// the CR name so it works on the annotation path, which runs before any read.
func (r *BackupReconciler) releaseMongoDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, sourceBackupName, reason string) (ctrl.Result, error) {
logger := getLogger(ctx)
live := &psmdbtypes.PerconaServerMongoDBBackup{}
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: sourceBackupName}, live); err == nil {
if controllerutil.ContainsFinalizer(live, psmdbDeleteBackupFinalizer) {
base := live.DeepCopy()
controllerutil.RemoveFinalizer(live, psmdbDeleteBackupFinalizer)
_ = r.Patch(ctx, live, client.MergeFrom(base))
}
if live.DeletionTimestamp.IsZero() {
_ = r.Delete(ctx, live)
}
err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: sourceBackupName}, live)
if err != nil || !controllerutil.ContainsFinalizer(live, psmdbDeleteBackupFinalizer) {
// Legacy backup (no delete-backup finalizer) or an unreadable CR: nothing
// of the driver's to strip or delete. Leave the operator CR untouched and
// release the Cozystack Backup.
logger.Debug("releasing MongoDB Backup; no driver-owned archive to prune",
"backup", backup.Name, "sourceBackup", sourceBackupName, "reason", reason)
return ctrl.Result{}, nil
}
getLogger(ctx).Info("releasing MongoDB Backup without pruning its object",
base := live.DeepCopy()
controllerutil.RemoveFinalizer(live, psmdbDeleteBackupFinalizer)
if perr := r.Patch(ctx, live, client.MergeFrom(base)); perr != nil {
// The release still proceeds (an unreachable CR must not wedge the
// Backup), but the CR keeps the finalizer and can pin its namespace in
// Terminating — surface which object so an operator can find it.
logger.Info("released MongoDB Backup but could not strip the delete-backup finalizer; the operator CR may hold its namespace in Terminating",
"backup", backup.Name, "sourceBackup", sourceBackupName, "reason", reason, "error", perr)
if r.Recorder != nil {
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "FinalizerNotStripped",
"released Backup but could not strip the delete-backup finalizer from PerconaServerMongoDBBackup %s (%v); it may keep the namespace in Terminating", sourceBackupName, perr)
}
return ctrl.Result{}, nil
}
if live.DeletionTimestamp.IsZero() {
_ = r.Delete(ctx, live)
}
logger.Info("releasing MongoDB Backup, leaving its shared-bucket object for a lifecycle policy",
"backup", backup.Name, "sourceBackup", sourceBackupName, "reason", reason)
if r.Recorder != nil {
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted",
@@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"testing"
corev1 "k8s.io/api/core/v1"
@@ -1507,4 +1508,57 @@ func TestCleanupMongoDBBackup(t *testing.T) {
t.Fatalf("NoMatchError must be treated as gone and release, got res=%+v err=%v", res, err)
}
})
// The annotation reaches releaseMongoDBCleanup before the ownership check, so
// the release must itself refuse to touch a CR the driver does not own — the
// legacy operator-side backup record is what a psmdb restore-by-name resolves
// against, and the RBAC comment and docs both promise it is never deleted.
t.Run("legacy backup with the skip annotation keeps its operator CR", func(t *testing.T) {
c := newMongoDBStrategyTestClient(t, opBackup(false), liveNamespace)
r := &BackupReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
res, err := r.cleanupMongoDBBackup(context.Background(), newBackup(map[string]string{psmdbSkipArtifactCleanupAnnotation: "true"}))
if err != nil || res.RequeueAfter != 0 {
t.Fatalf("legacy + annotation must release, got res=%+v err=%v", res, err)
}
got := getOp(t, c)
if !got.DeletionTimestamp.IsZero() {
t.Errorf("the escape hatch must not delete a legacy (unowned) operator CR")
}
})
// When the finalizer strip fails, the CR keeps percona.com/delete-backup and
// can pin its namespace in Terminating; the release still proceeds, but it
// must say which object is stuck rather than emit the success-case Event.
t.Run("a failed finalizer strip surfaces the stuck object", func(t *testing.T) {
s := runtime.NewScheme()
_ = scheme.AddToScheme(s)
_ = backupsv1alpha1.AddToScheme(s)
_ = psmdbtypes.AddToScheme(s)
c := clientfake.NewClientBuilder().
WithScheme(s).
WithObjects(opBackup(true)).
WithInterceptorFuncs(interceptor.Funcs{
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if _, ok := obj.(*psmdbtypes.PerconaServerMongoDBBackup); ok {
return apierrors.NewServiceUnavailable("apiserver down")
}
return cl.Patch(ctx, obj, patch, opts...)
},
}).
Build()
rec := record.NewFakeRecorder(10)
r := &BackupReconciler{Client: c, Recorder: rec}
res, err := r.cleanupMongoDBBackup(context.Background(), newBackup(map[string]string{psmdbSkipArtifactCleanupAnnotation: "true"}))
if err != nil || res.RequeueAfter != 0 {
t.Fatalf("release must proceed even when the strip fails, got res=%+v err=%v", res, err)
}
select {
case ev := <-rec.Events:
if !strings.Contains(ev, "FinalizerNotStripped") || !strings.Contains(ev, "op-bk") {
t.Errorf("expected a FinalizerNotStripped event naming the object, got %q", ev)
}
default:
t.Errorf("expected an event surfacing the stuck finalizer, got none")
}
})
}