mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): retry a failed read of the platform backup credentials
The MariaDB cleanup reaches the platform bucket through the platform's current endpoint so that a Backup taken before the endpoint moved is still deleted. A failed read of the platform credentials made it fall back to the endpoint recorded on the operator Backup, which may be the old address: there the dump is not found, a missing dump counts as already deleted, and the operator Backup that pointed at the real dump was deleted with a success log. A read that fails with an API error now holds the Backup and retries. A missing or malformed platform Secret still falls back, since it is a standing state that a retry would not change. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
37a74e5939
commit
bbeca34535
2 files changed
+38
-2
No files matched your search
@@ -220,12 +220,18 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv
|
||||
// Secret the platform credentials are projected to, is reached as the
|
||||
// platform reaches it now, so a Backup taken before the endpoint moved is
|
||||
// still deleted. Any other bucket, or the platform bucket while the
|
||||
// platform credentials cannot be read, is reached as op recorded it; a
|
||||
// delete that fails there is handled by retryMariaDBCleanup like any other.
|
||||
// platform credentials are missing or malformed, is reached as op recorded
|
||||
// it; a delete that fails there is handled by retryMariaDBCleanup like any
|
||||
// other. A failed read of the platform credentials is retried instead: the
|
||||
// recorded endpoint may be the one the platform moved off, where the dump
|
||||
// would be found missing and reported deleted.
|
||||
func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) {
|
||||
if r.CredentialsConfig.IsEnabled() {
|
||||
conn, bucket, err := r.platformS3Connection(ctx)
|
||||
var projErr *ProjectionError
|
||||
switch {
|
||||
case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError:
|
||||
return s3Connection{}, err
|
||||
case err != nil:
|
||||
getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err)
|
||||
case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName:
|
||||
|
||||
@@ -869,6 +869,36 @@ func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperator
|
||||
}
|
||||
}
|
||||
|
||||
// A missing or malformed platform Secret is a standing state, and the
|
||||
// operator Backup's own credentials are the best left. A failed read says
|
||||
// nothing about the Secret: falling back then can list a moved endpoint's
|
||||
// old address, find no dump there and report it deleted.
|
||||
func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}}
|
||||
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
|
||||
platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
||||
if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" {
|
||||
return errors.New("apiserver unavailable")
|
||||
}
|
||||
return c.Get(ctx, key, obj, opts...)
|
||||
},
|
||||
})
|
||||
r, rec := cleanupReconcilerWithClient(c, bucket)
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || !requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err)
|
||||
}
|
||||
if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) {
|
||||
t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client))
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
Reference in new issue
Block a user