From bbeca3453534124975682bda008f0d025aaed65c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:07:03 +0300 Subject: [PATCH] fix(backups): retry a failed read of the platform backup credentials The MariaDB cleanup reaches the platform bucket through the platform's current endpoint so that a Backup taken before the endpoint moved is still deleted. A failed read of the platform credentials made it fall back to the endpoint recorded on the operator Backup, which may be the old address: there the dump is not found, a missing dump counts as already deleted, and the operator Backup that pointed at the real dump was deleted with a success log. A read that fails with an API error now holds the Backup and retries. A missing or malformed platform Secret still falls back, since it is a standing state that a retry would not change. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 10 +++++-- .../mariadbstrategy_cleanup_test.go | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 531b806c9..b4617918e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -220,12 +220,18 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv // Secret the platform credentials are projected to, is reached as the // platform reaches it now, so a Backup taken before the endpoint moved is // still deleted. Any other bucket, or the platform bucket while the -// platform credentials cannot be read, is reached as op recorded it; a -// delete that fails there is handled by retryMariaDBCleanup like any other. +// platform credentials are missing or malformed, is reached as op recorded +// it; a delete that fails there is handled by retryMariaDBCleanup like any +// other. A failed read of the platform credentials is retried instead: the +// recorded endpoint may be the one the platform moved off, where the dump +// would be found missing and reported deleted. func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { if r.CredentialsConfig.IsEnabled() { conn, bucket, err := r.platformS3Connection(ctx) + var projErr *ProjectionError switch { + case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError: + return s3Connection{}, err case err != nil: getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 0ce6b2d01..1eecfeaff 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -869,6 +869,36 @@ func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperator } } +// A missing or malformed platform Secret is a standing state, and the +// operator Backup's own credentials are the best left. A failed read says +// nothing about the Secret: falling back then can list a moved endpoint's +// old address, find no dump there and report it deleted. +func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())