From 5f636ce62469aa313a3f1a67cba4e72dbf8f4d87 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:43:14 +0300 Subject: [PATCH] fix(clickhouse): do not hold strategy jobs on a credentials stamp that cannot expire The projector rewrites the credentials-updated-at stamp only when the projected content changes, so a stamp the hold cannot use stays put. An unreadable one failed every reconcile with no condition on the job, and one far in the future held each job again on every reconcile; in both cases every ClickHouse backup and restore in the namespace stopped for good. Such a stamp is now logged and ignored. A stamp only slightly ahead, as a skewed clock writes it, still waits, and that wait ends. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../altinitystrategy_controller.go | 10 ++++- .../altinitystrategy_controller_test.go | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index a99801ac6..c3c609350 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -102,9 +102,17 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName if raw == "" { return 0, nil } + // The projector rewrites the stamp only when the content changes, so a + // stamp this hold cannot use stays as it is: holding on it would block + // every Job in the namespace for good. updatedAt, err := time.Parse(time.RFC3339, raw) if err != nil { - return 0, fmt.Errorf("parse %s on Secret %s/%s: %w", credentialsUpdatedAtAnnotation, namespace, secretName, err) + getLogger(ctx).Error(err, "ignoring unreadable credentials stamp", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation) + return 0, nil + } + if updatedAt.After(time.Now().Add(altinityCredentialsPropagation)) { + getLogger(ctx).Error(nil, "ignoring a credentials stamp further ahead than clock skew explains", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation, "value", raw) + return 0, nil } return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil } diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 75564538b..717b425d1 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -896,3 +896,44 @@ func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) { t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) } } + +func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp}, + }, + } +} + +// The stamp is only ever rewritten when the projected content changes, so a +// value the hold cannot use must not hold the job: an unreadable one would +// fail every reconcile, and one far in the future would wait again on each. +// A stamp a little ahead, as a skewed clock writes it, still waits, but only +// until that stamp plus the propagation time. +func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) { + cases := []struct { + name string + stamp string + wantMin time.Duration + wantMax time.Duration + }{ + {name: "unreadable", stamp: "garbage"}, + {name: "far in the future", stamp: "3000-01-01T00:00:00Z"}, + {name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339), + wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build() + wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job") + if err != nil { + t.Fatalf("credentialsPropagationWait: %v", err) + } + if wait < tc.wantMin || wait > tc.wantMax { + t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax) + } + }) + } +}