From 1291e616c700def517907d072289e9970b8e694c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 16:39:02 +0300 Subject: [PATCH] fix(clickhouse): give every backup run an archive of its own An archive was named from the application and the BackupJob's start time, which has one-second resolution. Two runs of one application started in the same second, as two Plans on the same schedule are, wrote the same archive. The strategy takes a finished upload of its archive as an earlier attempt of its own run, so the later run could succeed on the other run's upload and record an archive missing any table created since. The BackupJob's UID now ends the name. It goes after the timestamp so names keep sorting chronologically, and the restore of a Backup that records no archive, which accepts only a bare timestamp after the release prefix, never picks one of these. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/altinity-backup-strategy.bats | 1 + .../altinitystrategy_controller.go | 14 +++--- .../altinitystrategy_controller_test.go | 44 +++++++++++++++---- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 46cf62d9a..d26b22ee4 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -393,6 +393,7 @@ run_script() { remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \ clickhouse-myapp-20260304T050710Z-copy \ + clickhouse-myapp-20260304T050720Z-0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f \ clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" out=$(run_script "$tmp" 2>&1) grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index a0868ec38..1aec3a585 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -117,12 +117,16 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string { } // altinityArchiveName mints the archive name for one BackupJob from its -// persisted StartedAt, so every reconcile and every Pod retry of the Job -// writes the same archive. It keeps the clickhouse-- -// shape, because a Backup that records no archive is restored by the -// timestamp in that shape. +// persisted StartedAt and its UID, so every reconcile and every Pod retry of +// the Job writes the same archive and no other run writes it. StartedAt has +// one-second resolution, and two Plans on one schedule start their runs in +// the same second; the strategy takes a finished upload of its archive as an +// earlier attempt of its own run, so a shared name would let one run report +// the other's archive. The UID goes after the timestamp so names still sort +// chronologically, and the restore of a Backup that records no archive, which +// accepts only clickhouse--, never picks one of these. func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { - return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout)) + return fmt.Sprintf("clickhouse-%s-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout), j.UID) } // altinityJobCarriesArchive reports whether the Job's Pod was rendered with diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index e5f1e1695..18cead060 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -730,14 +731,41 @@ func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity { func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) { startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600))) j := &backupsv1alpha1.BackupJob{ - Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, - Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, } - if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want { + if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z-0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"; got != want { t.Errorf("altinityArchiveName = %q, want %q", got, want) } } +// TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond pins that two +// BackupJobs of one application started within the same second, as two Plans +// on the same schedule are, write different archives. With a shared name, the +// strategy takes the other run's finished upload for an earlier attempt of its +// own and records an archive missing what changed since that upload. +func TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + run := func(name, uid string) *backupsv1alpha1.BackupJob { + return &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test", UID: types.UID(uid)}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + } + daily := altinityArchiveName(run("daily-1", "11111111-1111-1111-1111-111111111111")) + weekly := altinityArchiveName(run("weekly-1", "22222222-2222-2222-2222-222222222222")) + if daily == weekly { + t.Fatalf("two runs started in the same second share archive %q", daily) + } + for _, name := range []string{daily, weekly} { + if !strings.HasPrefix(name, "clickhouse-ch-test-20260304T050607Z-") { + t.Errorf("archive %q does not start with the application and start time, so names stop sorting chronologically", name) + } + } +} + // TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob // through Job creation and completion in separate reconciles. The archive // name has to come out of the persisted StartedAt both times: the Pod is @@ -748,7 +776,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -765,7 +793,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) ctx := context.Background() - const want = "clickhouse-ch-test-20260304T050607Z" + const want = "clickhouse-ch-test-20260304T050607Z-test-bj-uid" if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { t.Fatalf("reconcileAltinity() creating the Job: %v", err) @@ -869,13 +897,13 @@ func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { // A Backup recording this run's own archive is the one an earlier reconcile // created before its status write was lost, and is adopted. func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { - const thisRun = "clickhouse-ch-test-20260305T050607Z" + const thisRun = "clickhouse-ch-test-20260305T050607Z-test-bj-uid" cases := []struct { name string existing string // archive the existing Backup records; empty for none wantSucceeded bool }{ - {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"}, + {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z-earlier-bj-uid"}, {name: "earlier run's Backup records no archive", existing: ""}, {name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true}, } @@ -885,7 +913,7 @@ func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup",