fix(clickhouse): give every backup run an archive of its own

An archive was named from the application and the BackupJob's start
time, which has one-second resolution. Two runs of one application
started in the same second, as two Plans on the same schedule are,
wrote the same archive. The strategy takes a finished upload of its
archive as an earlier attempt of its own run, so the later run could
succeed on the other run's upload and record an archive missing any
table created since.

The BackupJob's UID now ends the name. It goes after the timestamp so
names keep sorting chronologically, and the restore of a Backup that
records no archive, which accepts only a bare timestamp after the
release prefix, never picks one of these.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-09 16:39:47 +03:00
1 parent d530ca8cf5
commit 1291e616c7
3 files changed
+46 -13

No files matched your search

+1
View File
@@ -393,6 +393,7 @@ run_script() {
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
"clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \
clickhouse-myapp-20260304T050710Z-copy \
clickhouse-myapp-20260304T050720Z-0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f \
clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
out=$(run_script "$tmp" 2>&1)
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
@@ -117,12 +117,16 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string {
}
// altinityArchiveName mints the archive name for one BackupJob from its
// persisted StartedAt, so every reconcile and every Pod retry of the Job
// writes the same archive. It keeps the clickhouse-<release>-<timestamp>
// shape, because a Backup that records no archive is restored by the
// timestamp in that shape.
// persisted StartedAt and its UID, so every reconcile and every Pod retry of
// the Job writes the same archive and no other run writes it. StartedAt has
// one-second resolution, and two Plans on one schedule start their runs in
// the same second; the strategy takes a finished upload of its archive as an
// earlier attempt of its own run, so a shared name would let one run report
// the other's archive. The UID goes after the timestamp so names still sort
// chronologically, and the restore of a Backup that records no archive, which
// accepts only clickhouse-<release>-<timestamp>, never picks one of these.
func altinityArchiveName(j *backupsv1alpha1.BackupJob) string {
return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout))
return fmt.Sprintf("clickhouse-%s-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout), j.UID)
}
// altinityJobCarriesArchive reports whether the Job's Pod was rendered with
@@ -14,6 +14,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
dynamicfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
@@ -730,14 +731,41 @@ func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity {
func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) {
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600)))
j := &backupsv1alpha1.BackupJob{
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"},
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
}
if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want {
if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z-0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"; got != want {
t.Errorf("altinityArchiveName = %q, want %q", got, want)
}
}
// TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond pins that two
// BackupJobs of one application started within the same second, as two Plans
// on the same schedule are, write different archives. With a shared name, the
// strategy takes the other run's finished upload for an earlier attempt of its
// own and records an archive missing what changed since that upload.
func TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond(t *testing.T) {
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
run := func(name, uid string) *backupsv1alpha1.BackupJob {
return &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test", UID: types.UID(uid)},
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
}
}
daily := altinityArchiveName(run("daily-1", "11111111-1111-1111-1111-111111111111"))
weekly := altinityArchiveName(run("weekly-1", "22222222-2222-2222-2222-222222222222"))
if daily == weekly {
t.Fatalf("two runs started in the same second share archive %q", daily)
}
for _, name := range []string{daily, weekly} {
if !strings.HasPrefix(name, "clickhouse-ch-test-20260304T050607Z-") {
t.Errorf("archive %q does not start with the application and start time, so names stop sorting chronologically", name)
}
}
}
// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob
// through Job creation and completion in separate reconciles. The archive
// name has to come out of the persisted StartedAt both times: the Pod is
@@ -748,7 +776,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) {
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
backupJob := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
BackupClassName: "clickhouse-backup",
@@ -765,7 +793,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) {
}
r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
ctx := context.Background()
const want = "clickhouse-ch-test-20260304T050607Z"
const want = "clickhouse-ch-test-20260304T050607Z-test-bj-uid"
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
@@ -869,13 +897,13 @@ func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) {
// A Backup recording this run's own archive is the one an earlier reconcile
// created before its status write was lost, and is adopted.
func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) {
const thisRun = "clickhouse-ch-test-20260305T050607Z"
const thisRun = "clickhouse-ch-test-20260305T050607Z-test-bj-uid"
cases := []struct {
name string
existing string // archive the existing Backup records; empty for none
wantSucceeded bool
}{
{name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"},
{name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z-earlier-bj-uid"},
{name: "earlier run's Backup records no archive", existing: ""},
{name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true},
}
@@ -885,7 +913,7 @@ func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) {
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC))
backupJob := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
BackupClassName: "clickhouse-backup",