fix(backups): retry a failed read of the platform backup credentials

The MariaDB cleanup reaches the platform bucket through the platform's
current endpoint so that a Backup taken before the endpoint moved is
still deleted. A failed read of the platform credentials made it fall
back to the endpoint recorded on the operator Backup, which may be the
old address: there the dump is not found, a missing dump counts as
already deleted, and the operator Backup that pointed at the real dump
was deleted with a success log.

A read that fails with an API error now holds the Backup and retries.
A missing or malformed platform Secret still falls back, since it is a
standing state that a retry would not change.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-11 15:07:03 +03:00
1 parent 37a74e5939
commit bbeca34535
2 files changed
+38 -2

No files matched your search

@@ -220,12 +220,18 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv
// Secret the platform credentials are projected to, is reached as the
// platform reaches it now, so a Backup taken before the endpoint moved is
// still deleted. Any other bucket, or the platform bucket while the
// platform credentials cannot be read, is reached as op recorded it; a
// delete that fails there is handled by retryMariaDBCleanup like any other.
// platform credentials are missing or malformed, is reached as op recorded
// it; a delete that fails there is handled by retryMariaDBCleanup like any
// other. A failed read of the platform credentials is retried instead: the
// recorded endpoint may be the one the platform moved off, where the dump
// would be found missing and reported deleted.
func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) {
if r.CredentialsConfig.IsEnabled() {
conn, bucket, err := r.platformS3Connection(ctx)
var projErr *ProjectionError
switch {
case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError:
return s3Connection{}, err
case err != nil:
getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err)
case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName:
@@ -869,6 +869,36 @@ func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperator
}
}
// A missing or malformed platform Secret is a standing state, and the
// operator Backup's own credentials are the best left. A failed read says
// nothing about the Secret: falling back then can list a moved endpoint's
// old address, find no dump there and report it deleted.
func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) {
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}}
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" {
return errors.New("apiserver unavailable")
}
return c.Get(ctx, key, obj, opts...)
},
})
r, rec := cleanupReconcilerWithClient(c, bucket)
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
if err != nil || !requeues(res) {
t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err)
}
if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) {
t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client))
}
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") {
t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events)
}
}
func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) {
bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")}
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())