fix(clickhouse): do not hold strategy jobs on a credentials stamp that cannot expire

The projector rewrites the credentials-updated-at stamp only when the
projected content changes, so a stamp the hold cannot use stays put.
An unreadable one failed every reconcile with no condition on the job,
and one far in the future held each job again on every reconcile; in
both cases every ClickHouse backup and restore in the namespace stopped
for good. Such a stamp is now logged and ignored. A stamp only slightly
ahead, as a skewed clock writes it, still waits, and that wait ends.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-04 19:43:14 +03:00
1 parent 38b00730ff
commit 5f636ce624
2 files changed
+50 -1

No files matched your search

@@ -102,9 +102,17 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName
if raw == "" {
return 0, nil
}
// The projector rewrites the stamp only when the content changes, so a
// stamp this hold cannot use stays as it is: holding on it would block
// every Job in the namespace for good.
updatedAt, err := time.Parse(time.RFC3339, raw)
if err != nil {
return 0, fmt.Errorf("parse %s on Secret %s/%s: %w", credentialsUpdatedAtAnnotation, namespace, secretName, err)
getLogger(ctx).Error(err, "ignoring unreadable credentials stamp", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation)
return 0, nil
}
if updatedAt.After(time.Now().Add(altinityCredentialsPropagation)) {
getLogger(ctx).Error(nil, "ignoring a credentials stamp further ahead than clock skew explains", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation, "value", raw)
return 0, nil
}
return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil
}
@@ -896,3 +896,44 @@ func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) {
t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
}
}
func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: "cozy-backups-creds",
Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp},
},
}
}
// The stamp is only ever rewritten when the projected content changes, so a
// value the hold cannot use must not hold the job: an unreadable one would
// fail every reconcile, and one far in the future would wait again on each.
// A stamp a little ahead, as a skewed clock writes it, still waits, but only
// until that stamp plus the propagation time.
func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) {
cases := []struct {
name string
stamp string
wantMin time.Duration
wantMax time.Duration
}{
{name: "unreadable", stamp: "garbage"},
{name: "far in the future", stamp: "3000-01-01T00:00:00Z"},
{name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339),
wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build()
wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job")
if err != nil {
t.Fatalf("credentialsPropagationWait: %v", err)
}
if wait < tc.wantMin || wait > tc.wantMax {
t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax)
}
})
}
}