fix(clickhouse): stop reporting a credentials wait once the strategy job runs

The hold sets Ready=False/WaitingForCredentials with a countdown, and
nothing removed it once the Job was created, so a backup or restore
that ran for minutes kept saying it was still waiting. The condition is
dropped when the Job runs; a Ready condition set for another reason is
left alone.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-04 19:43:15 +03:00
1 parent 5f636ce624
commit 6345b939d2
2 files changed
+127 -1

No files matched your search

@@ -117,13 +117,25 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName
return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil
}
// clearCredentialsWait drops the Ready condition credentialsWaitCondition
// left behind, once the job no longer waits. It reports whether it did.
func clearCredentialsWait(conditions *[]metav1.Condition) bool {
ready := apimeta.FindStatusCondition(*conditions, "Ready")
if ready == nil || ready.Reason != credentialsWaitReason {
return false
}
return apimeta.RemoveStatusCondition(conditions, "Ready")
}
const credentialsWaitReason = "WaitingForCredentials"
// credentialsWaitCondition is the Ready condition of a job held back by
// credentialsPropagationWait.
func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition {
return metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "WaitingForCredentials",
Reason: credentialsWaitReason,
Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar",
wait.Round(time.Second), secretName),
}
@@ -336,6 +348,11 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
return r.markBackupJobFailed(ctx, j, message)
default:
if clearCredentialsWait(&j.Status.Conditions) {
if err := r.Status().Update(ctx, j); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{RequeueAfter: altinityPollInterval}, nil
}
}
@@ -741,6 +758,11 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
return r.markRestoreJobFailed(ctx, restoreJob, message)
default:
if clearCredentialsWait(&restoreJob.Status.Conditions) {
if err := r.Status().Update(ctx, restoreJob); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{RequeueAfter: altinityPollInterval}, nil
}
}
@@ -907,6 +907,100 @@ func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret {
}
}
func waitingForCredentials() []metav1.Condition {
return []metav1.Condition{credentialsWaitCondition("cozy-backups-creds", time.Minute)}
}
// Once the hold is over and the Job runs, a Ready condition still saying the
// job is waiting for credentials describes a state the job has left.
func TestReconcileAltinity_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) {
now := metav1.Now()
backupJob := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
BackupClassName: "clickhouse-backup",
},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Conditions: waitingForCredentials()},
}
r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"),
projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
WithStatusSubresource(&backupsv1alpha1.BackupJob{}))
r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
}
ctx := context.Background()
j := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil {
t.Fatalf("reconcileAltinity: %v", err)
}
jobs := &batchv1.JobList{}
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
t.Fatalf("list Jobs: %v", err)
}
if len(jobs.Items) != 1 {
t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
}
stored := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), stored); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" {
t.Errorf("stored Ready = %+v while the Job runs", ready)
}
}
func TestReconcileAltinityRestore_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) {
now := metav1.Now()
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
TakenAt: now,
},
}
restoreJob := &backupsv1alpha1.RestoreJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning,
Conditions: waitingForCredentials()},
}
_, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob,
projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}))
rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
ctx := context.Background()
rj := &backupsv1alpha1.RestoreJob{}
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil {
t.Fatalf("get RestoreJob: %v", err)
}
if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err != nil {
t.Fatalf("reconcileAltinityRestore: %v", err)
}
stored := &backupsv1alpha1.RestoreJob{}
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), stored); err != nil {
t.Fatalf("get RestoreJob: %v", err)
}
if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" {
t.Errorf("stored Ready = %+v while the Job runs", ready)
}
}
// The stamp is only ever rewritten when the projected content changes, so a
// value the hold cannot use must not hold the job: an unreadable one would
// fail every reconcile, and one far in the future would wait again on each.
@@ -937,3 +1031,13 @@ func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T)
})
}
}
func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) {
conditions := []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "StrategyNotReady"}}
if clearCredentialsWait(&conditions) {
t.Errorf("clearCredentialsWait reported clearing a Ready condition with reason StrategyNotReady")
}
if ready := meta.FindStatusCondition(conditions, "Ready"); ready == nil || ready.Reason != "StrategyNotReady" {
t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready)
}
}