mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
A backup's archive is named from the BackupJob's persisted StartedAt, and the restore of a Backup that records no archive picks one stamped no later than its takenAt. The credentials wait holds the Job after StartedAt is set and writes the job's status while it holds. Should a held reconcile restamp StartedAt, the Pod would write an archive the earlier reconciles never named; nothing caught that. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
1574 lines
63 KiB
Go
1574 lines
63 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
dynamicfake "k8s.io/client-go/dynamic/fake"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
|
|
|
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
)
|
|
|
|
// newAltinityClickHouseApp returns an unstructured ClickHouse app the
|
|
// driver's dynamic client can serve. The fake client matches GVR exactly,
|
|
// so the apiVersion/kind/group must align with what mockRESTMapper returns
|
|
// below.
|
|
func newAltinityClickHouseApp(name, namespace string) *unstructured.Unstructured {
|
|
u := &unstructured.Unstructured{}
|
|
u.SetGroupVersionKind(schema.GroupVersionKind{
|
|
Group: backupsv1alpha1.DefaultApplicationAPIGroup,
|
|
Version: "v1alpha1",
|
|
Kind: "ClickHouse",
|
|
})
|
|
u.SetName(name)
|
|
u.SetNamespace(namespace)
|
|
return u
|
|
}
|
|
|
|
// newAltinityTestEnv builds the scheme, dynamic client, REST mapper, and
|
|
// reconciler harness used across Altinity strategy unit tests.
|
|
func newAltinityTestEnv(t *testing.T, app *unstructured.Unstructured, builder *clientfake.ClientBuilder) (*BackupJobReconciler, *RestoreJobReconciler, *runtime.Scheme) {
|
|
t.Helper()
|
|
|
|
testScheme := runtime.NewScheme()
|
|
_ = scheme.AddToScheme(testScheme)
|
|
_ = backupsv1alpha1.AddToScheme(testScheme)
|
|
_ = strategyv1alpha1.AddToScheme(testScheme)
|
|
|
|
gvr := schema.GroupVersionResource{
|
|
Group: backupsv1alpha1.DefaultApplicationAPIGroup,
|
|
Version: "v1alpha1",
|
|
Resource: "clickhouses",
|
|
}
|
|
dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
|
|
testScheme,
|
|
map[schema.GroupVersionResource]string{gvr: "ClickHouseList"},
|
|
app,
|
|
)
|
|
mapping := &meta.RESTMapping{
|
|
Resource: gvr,
|
|
GroupVersionKind: app.GroupVersionKind(),
|
|
Scope: meta.RESTScopeNamespace,
|
|
}
|
|
restMapper := &mockRESTMapper{mapping: mapping}
|
|
|
|
c := builder.WithScheme(testScheme).
|
|
WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}).
|
|
Build()
|
|
|
|
return &BackupJobReconciler{
|
|
Client: c,
|
|
Interface: dynamicClient,
|
|
RESTMapper: restMapper,
|
|
Scheme: testScheme,
|
|
Recorder: record.NewFakeRecorder(10),
|
|
}, &RestoreJobReconciler{
|
|
Client: c,
|
|
Interface: dynamicClient,
|
|
RESTMapper: restMapper,
|
|
Scheme: testScheme,
|
|
Recorder: record.NewFakeRecorder(10),
|
|
}, testScheme
|
|
}
|
|
|
|
func newAltinityStrategy(name string) *strategyv1alpha1.Altinity {
|
|
return &strategyv1alpha1.Altinity{
|
|
ObjectMeta: metav1.ObjectMeta{Name: name},
|
|
Spec: strategyv1alpha1.AltinitySpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Containers: []corev1.Container{{
|
|
Name: "clickhouse-backup",
|
|
Image: "altinity/clickhouse-backup:test",
|
|
Args: []string{
|
|
"--release={{ .Release.Name }}",
|
|
"--mode={{ .Mode }}",
|
|
"--bucket={{ .Parameters.bucketName }}",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func newAltinityClickHouseRef(name string) corev1.TypedLocalObjectReference {
|
|
return corev1.TypedLocalObjectReference{
|
|
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
|
|
Kind: "ClickHouse",
|
|
Name: name,
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinity_CreatesBatchJob(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityStrategy("clickhouse-strategy")
|
|
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
}
|
|
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
|
}
|
|
|
|
r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
|
|
ctx := context.Background()
|
|
|
|
// First reconcile patches Status.StartedAt and requeues without
|
|
// touching the Job - mirrors the CNPG/Velero pattern that avoids
|
|
// carrying a stale ResourceVersion into the next status write in the
|
|
// same reconcile.
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() first call: %v", err)
|
|
}
|
|
// Refresh local copy and reconcile again - this is the call that
|
|
// materialises the Job once StartedAt is already set.
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
|
|
t.Fatalf("refresh BackupJob after first reconcile: %v", err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() second call: %v", err)
|
|
}
|
|
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list batch jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 batch/v1.Job, got %d", len(jobs.Items))
|
|
}
|
|
k8sJob := jobs.Items[0]
|
|
if got := k8sJob.Labels[backupsv1alpha1.OwningJobNameLabel]; got != backupJob.Name {
|
|
t.Errorf("expected owning label %q, got %q", backupJob.Name, got)
|
|
}
|
|
if got := k8sJob.Labels[altinityLabelMode]; got != altinityModeBackup {
|
|
t.Errorf("expected mode label %q, got %q", altinityModeBackup, got)
|
|
}
|
|
|
|
containerArgs := k8sJob.Spec.Template.Spec.Containers[0].Args
|
|
wantArgs := []string{"--release=ch-test", "--mode=backup", "--bucket=ch-bucket"}
|
|
for i, want := range wantArgs {
|
|
if i >= len(containerArgs) || containerArgs[i] != want {
|
|
t.Errorf("rendered args[%d]: want %q, got %q", i, want, containerArgs)
|
|
break
|
|
}
|
|
}
|
|
|
|
updated := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: backupJob.Namespace, Name: backupJob.Name}, updated); err != nil {
|
|
t.Fatalf("get backupjob: %v", err)
|
|
}
|
|
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
|
|
t.Errorf("expected phase Running after first reconcile, got %q", updated.Status.Phase)
|
|
}
|
|
if updated.Status.StartedAt == nil {
|
|
t.Error("expected StartedAt to be set")
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinity_CompletesAndCreatesBackup(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityStrategy("clickhouse-strategy")
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{
|
|
StartedAt: &now,
|
|
Phase: backupsv1alpha1.BackupJobPhaseRunning,
|
|
},
|
|
}
|
|
completedK8sJob := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: jobNameForBackupJob(backupJob),
|
|
Namespace: backupJob.Namespace,
|
|
Labels: map[string]string{
|
|
backupsv1alpha1.OwningJobNameLabel: backupJob.Name,
|
|
backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace,
|
|
},
|
|
},
|
|
Status: batchv1.JobStatus{
|
|
CompletionTime: &now,
|
|
Conditions: []batchv1.JobCondition{
|
|
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue},
|
|
},
|
|
},
|
|
}
|
|
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
|
}
|
|
|
|
r, _, _ := newAltinityTestEnv(t, app,
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() error = %v", err)
|
|
}
|
|
|
|
updated := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: backupJob.Namespace, Name: backupJob.Name}, updated); err != nil {
|
|
t.Fatalf("get backupjob: %v", err)
|
|
}
|
|
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded {
|
|
t.Errorf("expected phase Succeeded, got %q", updated.Status.Phase)
|
|
}
|
|
if updated.Status.BackupRef == nil || updated.Status.BackupRef.Name == "" {
|
|
t.Fatalf("expected BackupRef to be set, got %v", updated.Status.BackupRef)
|
|
}
|
|
if updated.Status.CompletedAt == nil {
|
|
t.Error("expected CompletedAt to be set")
|
|
}
|
|
|
|
created := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: backupJob.Namespace, Name: updated.Status.BackupRef.Name}, created); err != nil {
|
|
t.Fatalf("get Backup: %v", err)
|
|
}
|
|
if created.Spec.StrategyRef.Name != "clickhouse-strategy" {
|
|
t.Errorf("expected StrategyRef name 'clickhouse-strategy', got %q", created.Spec.StrategyRef.Name)
|
|
}
|
|
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
|
|
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
|
|
}
|
|
// Status.Phase is intentionally NOT asserted here. The driver creates
|
|
// the Backup with Phase=Ready in its in-memory struct, but Backup has
|
|
// the status subresource enabled so the apiserver drops status on
|
|
// Create. The fake client's WithStatusSubresource registration mirrors
|
|
// that behaviour, and this driver does not own status promotion - that
|
|
// is the BackupReconciler's job (see internal/backupcontroller/
|
|
// backup_controller.go). Asserting Ready here would test the in-memory
|
|
// view of the create payload instead of the contract this driver
|
|
// publishes.
|
|
}
|
|
|
|
func TestReconcileAltinity_FailsOnJobFailed(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityStrategy("clickhouse-strategy")
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
failedK8sJob := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: jobNameForBackupJob(backupJob),
|
|
Namespace: backupJob.Namespace,
|
|
Labels: map[string]string{
|
|
backupsv1alpha1.OwningJobNameLabel: backupJob.Name,
|
|
backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace,
|
|
},
|
|
},
|
|
Status: batchv1.JobStatus{
|
|
Conditions: []batchv1.JobCondition{
|
|
{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Message: "BackoffLimitExceeded"},
|
|
},
|
|
},
|
|
}
|
|
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
|
|
r, _, _ := newAltinityTestEnv(t, app,
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, failedK8sJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() error = %v", err)
|
|
}
|
|
|
|
updated := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: backupJob.Namespace, Name: backupJob.Name}, updated); err != nil {
|
|
t.Fatalf("get backupjob: %v", err)
|
|
}
|
|
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Errorf("expected phase Failed, got %q", updated.Status.Phase)
|
|
}
|
|
if updated.Status.Message == "" {
|
|
t.Error("expected failure message to be set")
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinityRestore_CreatesBatchJobInTargetNamespace(t *testing.T) {
|
|
targetApp := newAltinityClickHouseApp("ch-restore", "tenant-test")
|
|
strategy := &strategyv1alpha1.Altinity{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "clickhouse-strategy"},
|
|
Spec: strategyv1alpha1.AltinitySpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Containers: []corev1.Container{{
|
|
Name: "clickhouse-backup",
|
|
Image: "altinity/clickhouse-backup:test",
|
|
Args: []string{
|
|
"--release={{ .Release.Name }}",
|
|
"--mode={{ .Mode }}",
|
|
"--source={{ .Backup.Name }}",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
DriverMetadata: map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"},
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: backup.Name},
|
|
TargetApplicationRef: &corev1.TypedLocalObjectReference{
|
|
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
|
|
Kind: "ClickHouse",
|
|
Name: "ch-restore",
|
|
},
|
|
},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
|
|
_, rr, _ := newAltinityTestEnv(t, targetApp,
|
|
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
|
|
t.Fatalf("reconcileAltinityRestore() error = %v", err)
|
|
}
|
|
|
|
jobs := &batchv1.JobList{}
|
|
if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list batch jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 restore Job, got %d", len(jobs.Items))
|
|
}
|
|
k8sJob := jobs.Items[0]
|
|
if got := k8sJob.Labels[altinityLabelMode]; got != altinityModeRestore {
|
|
t.Errorf("expected mode label %q, got %q", altinityModeRestore, got)
|
|
}
|
|
args := k8sJob.Spec.Template.Spec.Containers[0].Args
|
|
want := []string{"--release=ch-restore", "--mode=restore", "--source=ch-backup"}
|
|
for i, w := range want {
|
|
if i >= len(args) || args[i] != w {
|
|
t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinity_RenderedBackupJobOwnedByBackupJob locks in review
|
|
// Blocker 3: the rendered batch/v1.Job must carry an OwnerReferences entry
|
|
// pointing at the BackupJob so kube-gc collects it (and the running Pod)
|
|
// when the BackupJob is deleted. Before the fix the Job had no
|
|
// controllerRef; tenants who deleted a BackupJob mid-run leaked the Job.
|
|
func TestReconcileAltinity_RenderedBackupJobOwnedByBackupJob(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityStrategy("clickhouse-strategy")
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{
|
|
StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning,
|
|
},
|
|
}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
r, _, _ := newAltinityTestEnv(t, app,
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
|
|
ctx := context.Background()
|
|
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity: %v", err)
|
|
}
|
|
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list batch jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 backup Job, got %d", len(jobs.Items))
|
|
}
|
|
owners := jobs.Items[0].OwnerReferences
|
|
if len(owners) == 0 {
|
|
t.Fatal("rendered Job has no OwnerReferences (kube-gc would not collect on BackupJob delete)")
|
|
}
|
|
got := owners[0]
|
|
if got.Kind != "BackupJob" || got.Name != backupJob.Name || got.UID != backupJob.UID {
|
|
t.Errorf("OwnerRef does not point at BackupJob: %+v", got)
|
|
}
|
|
if got.Controller == nil || !*got.Controller {
|
|
t.Error("OwnerRef.Controller must be true so kube-gc treats this as the controlling owner")
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinityRestore_RenderedRestoreJobOwnedByRestoreJob mirrors
|
|
// the BackupJob test for the restore path: the rendered Job in the same
|
|
// namespace as the RestoreJob (the supported case - cross-namespace restore
|
|
// is rejected at the API level by TypedLocalObjectReference) must carry an
|
|
// OwnerReferences entry on the RestoreJob.
|
|
func TestReconcileAltinityRestore_RenderedRestoreJobOwnedByRestoreJob(t *testing.T) {
|
|
targetApp := newAltinityClickHouseApp("ch-restore", "tenant-test")
|
|
strategy := newAltinityStrategy("clickhouse-strategy")
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: backup.Name},
|
|
TargetApplicationRef: &corev1.TypedLocalObjectReference{
|
|
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
|
|
Kind: "ClickHouse",
|
|
Name: "ch-restore",
|
|
},
|
|
},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
_, rr, _ := newAltinityTestEnv(t, targetApp,
|
|
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
|
|
t.Fatalf("reconcileAltinityRestore: %v", err)
|
|
}
|
|
|
|
jobs := &batchv1.JobList{}
|
|
if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list batch jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 restore Job, got %d", len(jobs.Items))
|
|
}
|
|
owners := jobs.Items[0].OwnerReferences
|
|
if len(owners) == 0 {
|
|
t.Fatal("rendered restore Job has no OwnerReferences")
|
|
}
|
|
got := owners[0]
|
|
if got.Kind != "RestoreJob" || got.Name != restoreJob.Name || got.UID != restoreJob.UID {
|
|
t.Errorf("OwnerRef does not point at RestoreJob: %+v", got)
|
|
}
|
|
if got.Controller == nil || !*got.Controller {
|
|
t.Error("OwnerRef.Controller must be true")
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinityRestore_TargetNamespaceIsRestoreJobNamespace pins
|
|
// the invariant that the rendered restore Job always lives in the
|
|
// RestoreJob's own namespace, regardless of TargetApplicationRef shape.
|
|
// TargetApplicationRef is corev1.TypedLocalObjectReference (no Namespace
|
|
// field) and BackupRef is corev1.LocalObjectReference, so cross-namespace
|
|
// restore is not representable in the current API. Locking this in
|
|
// catches regressions if a future API change reintroduces a Namespace
|
|
// field on either ref - the helper that owns the OwnerReference (and
|
|
// kube-gc collection) assumes same-namespace and would silently leak Jobs
|
|
// otherwise.
|
|
func TestReconcileAltinityRestore_TargetNamespaceIsRestoreJobNamespace(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
targetRef *corev1.TypedLocalObjectReference
|
|
targetName string // app the dynamic client serves
|
|
}{
|
|
{
|
|
name: "nil targetRef -> in-place restore",
|
|
targetRef: nil,
|
|
targetName: "ch-source",
|
|
},
|
|
{
|
|
name: "name-only targetRef -> to-copy with default APIGroup/Kind",
|
|
targetRef: &corev1.TypedLocalObjectReference{
|
|
Name: "ch-copy",
|
|
},
|
|
targetName: "ch-copy",
|
|
},
|
|
{
|
|
name: "full triple targetRef -> to-copy explicit",
|
|
targetRef: &corev1.TypedLocalObjectReference{
|
|
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
|
|
Kind: "ClickHouse",
|
|
Name: "ch-copy",
|
|
},
|
|
targetName: "ch-copy",
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
targetApp := newAltinityClickHouseApp(tc.targetName, "tenant-foo")
|
|
strategy := newAltinityStrategy("clickhouse-strategy")
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-foo"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-source"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-foo"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: backup.Name},
|
|
TargetApplicationRef: tc.targetRef,
|
|
},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
_, rr, _ := newAltinityTestEnv(t, targetApp,
|
|
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
|
|
t.Fatalf("reconcileAltinityRestore: %v", err)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := rr.List(ctx, jobs); err != nil {
|
|
t.Fatalf("list batch jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 restore Job, got %d", len(jobs.Items))
|
|
}
|
|
if got := jobs.Items[0].Namespace; got != "tenant-foo" {
|
|
t.Errorf("rendered Job lives in %q, expected RestoreJob's namespace tenant-foo", got)
|
|
}
|
|
owners := jobs.Items[0].OwnerReferences
|
|
if len(owners) == 0 || owners[0].Kind != "RestoreJob" || owners[0].Name != restoreJob.Name {
|
|
t.Errorf("rendered Job missing controllerRef on RestoreJob: %+v", owners)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinityRestore_ExposesSourceApplicationRef pins the
|
|
// driver-side fix that lets restore strategies filter list/remote results
|
|
// by the *source* release name when restoring into a differently-named
|
|
// target (to-copy). Without `.Backup.ApplicationRef.Name` in the template
|
|
// context, the strategy script can only see `.Release.Name` (which is the
|
|
// destination) and any prefix-based filter against backup names ends up
|
|
// matching nothing - the script falls through to "no remote backup
|
|
// found" or, worse, blindly picks the latest archive in the bucket
|
|
// (regression covered by the multi-release isolation test in the bats
|
|
// e2e suite).
|
|
func TestReconcileAltinityRestore_ExposesSourceApplicationRef(t *testing.T) {
|
|
targetApp := newAltinityClickHouseApp("ch-restore", "tenant-foo")
|
|
strategy := &strategyv1alpha1.Altinity{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "clickhouse-strategy"},
|
|
Spec: strategyv1alpha1.AltinitySpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Containers: []corev1.Container{{
|
|
Name: "clickhouse-backup",
|
|
Image: "altinity/clickhouse-backup:test",
|
|
Args: []string{
|
|
"--target={{ .Release.Name }}",
|
|
"--source={{ .Backup.ApplicationRef.Name }}",
|
|
"--source-kind={{ .Backup.ApplicationRef.Kind }}",
|
|
},
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-foo"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-source"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-foo"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{
|
|
BackupRef: corev1.LocalObjectReference{Name: backup.Name},
|
|
TargetApplicationRef: &corev1.TypedLocalObjectReference{
|
|
APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup),
|
|
Kind: "ClickHouse",
|
|
Name: "ch-restore",
|
|
},
|
|
},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
|
|
_, rr, _ := newAltinityTestEnv(t, targetApp,
|
|
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
|
|
t.Fatalf("reconcileAltinityRestore: %v", err)
|
|
}
|
|
|
|
jobs := &batchv1.JobList{}
|
|
if err := rr.List(ctx, jobs, client.InNamespace("tenant-foo")); err != nil {
|
|
t.Fatalf("list batch jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("expected 1 restore Job, got %d", len(jobs.Items))
|
|
}
|
|
args := jobs.Items[0].Spec.Template.Spec.Containers[0].Args
|
|
want := []string{"--target=ch-restore", "--source=ch-source", "--source-kind=ClickHouse"}
|
|
for i, w := range want {
|
|
if i >= len(args) || args[i] != w {
|
|
t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackupParameters_RoundTrip(t *testing.T) {
|
|
backup := &backupsv1alpha1.Backup{
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
DriverMetadata: map[string]string{
|
|
altinityParamPrefix + "bucketName": "b1",
|
|
altinityParamPrefix + "endpoint": "https://s3",
|
|
altinityArchiveMetaKey: "clickhouse-ch-test-20260304T050607Z",
|
|
"job.batch/name": "ignored",
|
|
"": "ignored",
|
|
},
|
|
},
|
|
}
|
|
got := backupParameters(backup)
|
|
if got["bucketName"] != "b1" || got["endpoint"] != "https://s3" {
|
|
t.Errorf("backupParameters = %v", got)
|
|
}
|
|
if len(got) != 2 {
|
|
t.Errorf("non-parameter key leaked through: %v", got)
|
|
}
|
|
}
|
|
|
|
// newAltinityArchiveStrategy renders the archive name and the Backup's
|
|
// takenAt where a test can read them back off the Job.
|
|
func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity {
|
|
s := newAltinityStrategy(name)
|
|
s.Spec.Template.Spec.Containers[0].Args = []string{
|
|
"--mode={{ .Mode }}",
|
|
"--archive={{ .ArchiveName }}",
|
|
"--taken-at={{ .Backup.TakenAt }}",
|
|
"--bucket={{ .Parameters.bucketName }}",
|
|
}
|
|
return s
|
|
}
|
|
|
|
func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) {
|
|
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600)))
|
|
j := &backupsv1alpha1.BackupJob{
|
|
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
|
|
}
|
|
if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want {
|
|
t.Errorf("altinityArchiveName = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob
|
|
// through Job creation and completion in separate reconciles. The archive
|
|
// name has to come out of the persisted StartedAt both times: the Pod is
|
|
// told it on the first, the Backup records it on the second, and a restore
|
|
// later reads back exactly that archive.
|
|
func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
|
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
|
}
|
|
r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
|
|
ctx := context.Background()
|
|
const want = "clickhouse-ch-test-20260304T050607Z"
|
|
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
|
|
}
|
|
job := &batchv1.Job{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
|
|
t.Fatalf("get backup Job: %v", err)
|
|
}
|
|
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want {
|
|
t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, want)
|
|
}
|
|
|
|
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
|
|
if err := r.Status().Update(ctx, job); err != nil {
|
|
t.Fatalf("mark backup Job complete: %v", err)
|
|
}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
|
|
t.Fatalf("refresh BackupJob: %v", err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
|
|
}
|
|
|
|
created := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
|
|
t.Fatalf("get Backup: %v", err)
|
|
}
|
|
if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want {
|
|
t.Errorf("Backup records archive %q, want %q", got, want)
|
|
}
|
|
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
|
|
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose
|
|
// Pod never carried the driver's archive name: one created before the
|
|
// upgrade that introduced the names, or rendered from a strategy that names
|
|
// its own archives. Recording the name would send the restore after an
|
|
// archive nobody wrote; recording none leaves it the fallback for Backups
|
|
// without a name.
|
|
func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
|
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
completedK8sJob := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{Name: jobNameForBackupJob(backupJob), Namespace: backupJob.Namespace},
|
|
Spec: batchv1.JobSpec{
|
|
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Containers: []corev1.Container{{
|
|
Name: "clickhouse-backup",
|
|
Image: "altinity/clickhouse-backup:test",
|
|
Args: []string{`target="clickhouse-ch-test-$(date -u +%Y%m%dT%H%M%SZ)"`},
|
|
}},
|
|
}},
|
|
},
|
|
Status: batchv1.JobStatus{
|
|
Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}},
|
|
},
|
|
}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
|
}
|
|
r, _, _ := newAltinityTestEnv(t, app,
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() error = %v", err)
|
|
}
|
|
created := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
|
|
t.Fatalf("get Backup: %v", err)
|
|
}
|
|
if got, ok := created.Spec.DriverMetadata[altinityArchiveMetaKey]; ok {
|
|
t.Errorf("Backup records archive %q its Job was never given", got)
|
|
}
|
|
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
|
|
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
|
|
}
|
|
}
|
|
|
|
// TestReconcileAltinity_BackupNameAlreadyTaken covers a completed Job whose
|
|
// Backup name is already in use. A Backup outlives its BackupJob, so a
|
|
// BackupJob re-created under an earlier run's name finds that run's Backup,
|
|
// and claiming it would point the new run at an archive it did not write.
|
|
// A Backup recording this run's own archive is the one an earlier reconcile
|
|
// created before its status write was lost, and is adopted.
|
|
func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) {
|
|
const thisRun = "clickhouse-ch-test-20260305T050607Z"
|
|
cases := []struct {
|
|
name string
|
|
existing string // archive the existing Backup records; empty for none
|
|
wantSucceeded bool
|
|
}{
|
|
{name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"},
|
|
{name: "earlier run's Backup records no archive", existing: ""},
|
|
{name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
app := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
|
startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC))
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
|
|
if tc.existing != "" {
|
|
driverMD[altinityArchiveMetaKey] = tc.existing
|
|
}
|
|
existing := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: backupJob.Name, Namespace: backupJob.Namespace},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 22, 0, time.UTC)),
|
|
DriverMetadata: driverMD,
|
|
},
|
|
}
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
Parameters: map[string]string{"bucketName": "ch-bucket"},
|
|
}
|
|
r, _, _ := newAltinityTestEnv(t, app,
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, existing))
|
|
ctx := context.Background()
|
|
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
|
|
}
|
|
job := &batchv1.Job{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
|
|
t.Fatalf("get backup Job: %v", err)
|
|
}
|
|
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+thisRun {
|
|
t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, thisRun)
|
|
}
|
|
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
|
|
if err := r.Status().Update(ctx, job); err != nil {
|
|
t.Fatalf("mark backup Job complete: %v", err)
|
|
}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
|
|
t.Fatalf("refresh BackupJob: %v", err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
|
|
}
|
|
|
|
updated := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if tc.wantSucceeded {
|
|
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded {
|
|
t.Errorf("phase = %q (%s), want Succeeded", updated.Status.Phase, updated.Status.Message)
|
|
}
|
|
if updated.Status.BackupRef == nil || updated.Status.BackupRef.Name != existing.Name {
|
|
t.Errorf("BackupRef = %v, want %s", updated.Status.BackupRef, existing.Name)
|
|
}
|
|
} else {
|
|
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Errorf("phase = %q, want Failed: the run would claim a Backup recording %q instead of %q", updated.Status.Phase, tc.existing, thisRun)
|
|
}
|
|
if updated.Status.BackupRef != nil {
|
|
t.Errorf("BackupRef = %v, want none", updated.Status.BackupRef)
|
|
}
|
|
if !strings.Contains(updated.Status.Message, thisRun) {
|
|
t.Errorf("failure message %q does not name the archive this run wrote", updated.Status.Message)
|
|
}
|
|
}
|
|
kept := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(existing), kept); err != nil {
|
|
t.Fatalf("get existing Backup: %v", err)
|
|
}
|
|
if got := kept.Spec.DriverMetadata[altinityArchiveMetaKey]; got != tc.existing {
|
|
t.Errorf("existing Backup now records archive %q, want %q left in place", got, tc.existing)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
archive string // recorded on the Backup; empty for a Backup taken before names were recorded
|
|
wantArchive string
|
|
}{
|
|
{name: "recorded archive", archive: "clickhouse-ch-test-20260304T050607Z", wantArchive: "--archive=clickhouse-ch-test-20260304T050607Z"},
|
|
{name: "no recorded archive", archive: "", wantArchive: "--archive="},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
targetApp := newAltinityClickHouseApp("ch-test", "tenant-test")
|
|
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
|
|
driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
|
|
if tc.archive != "" {
|
|
driverMD[altinityArchiveMetaKey] = tc.archive
|
|
}
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 22, 0, time.FixedZone("UTC+3", 3*3600))),
|
|
DriverMetadata: driverMD,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
_, rr, _ := newAltinityTestEnv(t, targetApp,
|
|
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
|
|
ctx := context.Background()
|
|
|
|
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
|
|
t.Fatalf("reconcileAltinityRestore: %v", err)
|
|
}
|
|
job := &batchv1.Job{}
|
|
if err := rr.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForRestoreJob(restoreJob)}, job); err != nil {
|
|
t.Fatalf("get restore Job: %v", err)
|
|
}
|
|
args := job.Spec.Template.Spec.Containers[0].Args
|
|
want := []string{"--mode=restore", tc.wantArchive, "--taken-at=20260304T050622Z", "--bucket=ch-bucket"}
|
|
for i, w := range want {
|
|
if i >= len(args) || args[i] != w {
|
|
t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func projectedCredentials(namespace string, updatedAt time.Time) *corev1.Secret {
|
|
return &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: namespace,
|
|
Name: "cozy-backups-creds",
|
|
Annotations: map[string]string{credentialsUpdatedAtAnnotation: updatedAt.UTC().Format(time.RFC3339)},
|
|
},
|
|
}
|
|
}
|
|
|
|
// The ClickHouse sidecar reads the projected credentials from a mounted
|
|
// file that kubelet rewrites only some time after the Secret changes, so a
|
|
// strategy Job started right after a projection would reach a sidecar with
|
|
// no or stale credentials.
|
|
func TestReconcileAltinity_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
secret *corev1.Secret
|
|
jobExists bool
|
|
wantWait bool
|
|
}{
|
|
{name: "credentials changed just now", secret: projectedCredentials("tenant-test", time.Now()), wantWait: true},
|
|
{name: "credentials changed long ago", secret: projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))},
|
|
{name: "no projected credentials", secret: nil},
|
|
{name: "the Job already runs", secret: projectedCredentials("tenant-test", time.Now()), jobExists: true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now},
|
|
}
|
|
objs := []client.Object{backupJob, newAltinityStrategy("clickhouse-strategy")}
|
|
if tc.secret != nil {
|
|
objs = append(objs, tc.secret)
|
|
}
|
|
if tc.jobExists {
|
|
objs = append(objs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}})
|
|
}
|
|
r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), clientfake.NewClientBuilder().WithObjects(objs...))
|
|
r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
ctx := context.Background()
|
|
j := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
res, err := r.reconcileAltinity(ctx, j, resolved)
|
|
if err != nil {
|
|
t.Fatalf("reconcileAltinity: %v", err)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list Jobs: %v", err)
|
|
}
|
|
ready := meta.FindStatusCondition(j.Status.Conditions, "Ready")
|
|
if tc.wantWait {
|
|
if len(jobs.Items) != 0 {
|
|
t.Errorf("a strategy Job was created %d time(s) before the credentials could reach the sidecar", len(jobs.Items))
|
|
}
|
|
if res.RequeueAfter <= 0 || res.RequeueAfter > altinityCredentialsPropagation {
|
|
t.Errorf("RequeueAfter = %v, want within (0, %v]", res.RequeueAfter, altinityCredentialsPropagation)
|
|
}
|
|
if ready == nil || ready.Reason != "WaitingForCredentials" {
|
|
t.Errorf("Ready = %+v, want WaitingForCredentials", ready)
|
|
}
|
|
return
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Errorf("got %d strategy Jobs, want 1", len(jobs.Items))
|
|
}
|
|
if ready != nil && ready.Reason == "WaitingForCredentials" {
|
|
t.Errorf("Ready = %+v, the job should not wait", ready)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinityRestore_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) {
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
_, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
|
|
clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob,
|
|
projectedCredentials("tenant-test", time.Now())))
|
|
rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
ctx := context.Background()
|
|
rj := &backupsv1alpha1.RestoreJob{}
|
|
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil {
|
|
t.Fatalf("get RestoreJob: %v", err)
|
|
}
|
|
res, err := rr.reconcileAltinityRestore(ctx, rj, backup)
|
|
if err != nil {
|
|
t.Fatalf("reconcileAltinityRestore: %v", err)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list Jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Errorf("a restore Job was created before the credentials could reach the sidecar")
|
|
}
|
|
if res.RequeueAfter <= 0 {
|
|
t.Errorf("RequeueAfter = %v, want a wait", res.RequeueAfter)
|
|
}
|
|
if ready := meta.FindStatusCondition(rj.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCredentials" {
|
|
t.Errorf("Ready = %+v, want WaitingForCredentials", ready)
|
|
}
|
|
}
|
|
|
|
// Without credentials projection configured there is no Secret to wait
|
|
// for; client-go refuses a Get with an empty name, so it must not be asked.
|
|
func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) {
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now},
|
|
}
|
|
builder := clientfake.NewClientBuilder().
|
|
WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy")).
|
|
WithInterceptorFuncs(interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if key.Name == "" {
|
|
return errors.New("resource name may not be empty")
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder)
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
ctx := context.Background()
|
|
j := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity: %v", err)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list Jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
|
|
}
|
|
}
|
|
|
|
func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret {
|
|
return &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: namespace,
|
|
Name: "cozy-backups-creds",
|
|
Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp},
|
|
},
|
|
}
|
|
}
|
|
|
|
func waitingForCredentials() []metav1.Condition {
|
|
return []metav1.Condition{credentialsWaitCondition("cozy-backups-creds", time.Minute)}
|
|
}
|
|
|
|
// Once the hold is over and the Job runs, a Ready condition still saying the
|
|
// job is waiting for credentials describes a state the job has left.
|
|
func TestReconcileAltinity_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) {
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Conditions: waitingForCredentials()},
|
|
}
|
|
r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"),
|
|
projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
|
|
WithStatusSubresource(&backupsv1alpha1.BackupJob{}))
|
|
r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
ctx := context.Background()
|
|
j := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil {
|
|
t.Fatalf("reconcileAltinity: %v", err)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list Jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 1 {
|
|
t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
|
|
}
|
|
stored := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), stored); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" {
|
|
t.Errorf("stored Ready = %+v while the Job runs", ready)
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinityRestore_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) {
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning,
|
|
Conditions: waitingForCredentials()},
|
|
}
|
|
_, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
|
|
clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob,
|
|
projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
|
|
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}))
|
|
rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
ctx := context.Background()
|
|
rj := &backupsv1alpha1.RestoreJob{}
|
|
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil {
|
|
t.Fatalf("get RestoreJob: %v", err)
|
|
}
|
|
if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err != nil {
|
|
t.Fatalf("reconcileAltinityRestore: %v", err)
|
|
}
|
|
stored := &backupsv1alpha1.RestoreJob{}
|
|
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), stored); err != nil {
|
|
t.Fatalf("get RestoreJob: %v", err)
|
|
}
|
|
if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" {
|
|
t.Errorf("stored Ready = %+v while the Job runs", ready)
|
|
}
|
|
}
|
|
|
|
// The stamp is only ever rewritten when the projected content changes, so a
|
|
// value the hold cannot use must not hold the job: an unreadable one would
|
|
// fail every reconcile, and one far in the future would wait again on each.
|
|
// A stamp a little ahead, as a skewed clock writes it, still waits, but only
|
|
// until that stamp plus the propagation time.
|
|
func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
stamp string
|
|
wantMin time.Duration
|
|
wantMax time.Duration
|
|
}{
|
|
{name: "unreadable", stamp: "garbage"},
|
|
{name: "far in the future", stamp: "3000-01-01T00:00:00Z"},
|
|
{name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339),
|
|
wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build()
|
|
wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job")
|
|
if err != nil {
|
|
t.Fatalf("credentialsPropagationWait: %v", err)
|
|
}
|
|
if wait < tc.wantMin || wait > tc.wantMax {
|
|
t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) {
|
|
conditions := []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "StrategyNotReady"}}
|
|
if clearCredentialsWait(&conditions) {
|
|
t.Errorf("clearCredentialsWait reported clearing a Ready condition with reason StrategyNotReady")
|
|
}
|
|
if ready := meta.FindStatusCondition(conditions, "Ready"); ready == nil || ready.Reason != "StrategyNotReady" {
|
|
t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready)
|
|
}
|
|
}
|
|
|
|
// A read the hold cannot complete must not release the Job: starting it
|
|
// before the credentials reach the sidecar is the failure the hold prevents.
|
|
func TestReconcileAltinity_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
fail func(key client.ObjectKey, obj client.Object) bool
|
|
}{
|
|
{name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool {
|
|
_, isSecret := obj.(*corev1.Secret)
|
|
return isSecret && key.Name == "cozy-backups-creds"
|
|
}},
|
|
{name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool {
|
|
_, isJob := obj.(*batchv1.Job)
|
|
return isJob
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
now := metav1.Now()
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now},
|
|
}
|
|
builder := clientfake.NewClientBuilder().
|
|
WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"),
|
|
projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
|
|
WithInterceptorFuncs(interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if tc.fail(key, obj) {
|
|
return apierrors.NewInternalError(errors.New("etcd unavailable"))
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder)
|
|
r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
ctx := context.Background()
|
|
j := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, j, resolved); err == nil {
|
|
t.Errorf("reconcileAltinity returned no error although the %s could not be read", tc.name)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list Jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Errorf("a strategy Job was created although the %s could not be read", tc.name)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
fail func(key client.ObjectKey, obj client.Object) bool
|
|
}{
|
|
{name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool {
|
|
_, isSecret := obj.(*corev1.Secret)
|
|
return isSecret && key.Name == "cozy-backups-creds"
|
|
}},
|
|
{name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool {
|
|
_, isJob := obj.(*batchv1.Job)
|
|
return isJob
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
now := metav1.Now()
|
|
backup := &backupsv1alpha1.Backup{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
TakenAt: now,
|
|
},
|
|
}
|
|
restoreJob := &backupsv1alpha1.RestoreJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
|
|
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
|
}
|
|
builder := clientfake.NewClientBuilder().
|
|
WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob,
|
|
projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
|
|
WithInterceptorFuncs(interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if tc.fail(key, obj) {
|
|
return apierrors.NewInternalError(errors.New("etcd unavailable"))
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
_, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder)
|
|
rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
ctx := context.Background()
|
|
rj := &backupsv1alpha1.RestoreJob{}
|
|
if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil {
|
|
t.Fatalf("get RestoreJob: %v", err)
|
|
}
|
|
if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err == nil {
|
|
t.Errorf("reconcileAltinityRestore returned no error although the %s could not be read", tc.name)
|
|
}
|
|
jobs := &batchv1.JobList{}
|
|
if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
|
|
t.Fatalf("list Jobs: %v", err)
|
|
}
|
|
if len(jobs.Items) != 0 {
|
|
t.Errorf("a restore Job was created although the %s could not be read", tc.name)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A backup held for the credentials keeps the archive name it was given from
|
|
// its persisted StartedAt: the held reconciles write the job's status, and the
|
|
// Pod created after the hold, and the Backup recorded after it, must still
|
|
// name the same archive, which is stamped no later than the Backup's takenAt.
|
|
func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) {
|
|
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
|
|
backupJob := &backupsv1alpha1.BackupJob{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
|
|
Spec: backupsv1alpha1.BackupJobSpec{
|
|
ApplicationRef: newAltinityClickHouseRef("ch-test"),
|
|
BackupClassName: "clickhouse-backup",
|
|
},
|
|
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
|
|
}
|
|
secret := projectedCredentials("tenant-test", time.Now())
|
|
r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
|
|
clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityArchiveStrategy("clickhouse-strategy"), secret))
|
|
r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
|
|
resolved := &ResolvedBackupConfig{
|
|
StrategyRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
|
|
Kind: strategyv1alpha1.AltinityStrategyKind,
|
|
Name: "clickhouse-strategy",
|
|
},
|
|
}
|
|
ctx := context.Background()
|
|
const want = "clickhouse-ch-test-20260304T050607Z"
|
|
reconcile := func(step string) {
|
|
t.Helper()
|
|
j := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
|
|
t.Fatalf("%s: get BackupJob: %v", step, err)
|
|
}
|
|
if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil {
|
|
t.Fatalf("%s: reconcileAltinity: %v", step, err)
|
|
}
|
|
}
|
|
|
|
reconcile("held")
|
|
job := &batchv1.Job{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); !apierrors.IsNotFound(err) {
|
|
t.Fatalf("the Job exists while the credentials wait should hold it: %v", err)
|
|
}
|
|
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil {
|
|
t.Fatalf("get projected Secret: %v", err)
|
|
}
|
|
secret.Annotations[credentialsUpdatedAtAnnotation] = time.Now().Add(-altinityCredentialsPropagation - time.Second).UTC().Format(time.RFC3339)
|
|
if err := r.Update(ctx, secret); err != nil {
|
|
t.Fatalf("age the credentials stamp: %v", err)
|
|
}
|
|
reconcile("after the wait")
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
|
|
t.Fatalf("get backup Job: %v", err)
|
|
}
|
|
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want {
|
|
t.Fatalf("backup Pod was rendered with %q after the wait, want --archive=%s", got, want)
|
|
}
|
|
|
|
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
|
|
if err := r.Status().Update(ctx, job); err != nil {
|
|
t.Fatalf("mark backup Job complete: %v", err)
|
|
}
|
|
reconcile("completed")
|
|
created := &backupsv1alpha1.Backup{}
|
|
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
|
|
t.Fatalf("get Backup: %v", err)
|
|
}
|
|
if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want {
|
|
t.Errorf("Backup records archive %q, want %q", got, want)
|
|
}
|
|
if stamp := created.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout); stamp < "20260304T050607Z" {
|
|
t.Errorf("Backup takenAt %s is earlier than its archive's stamp", stamp)
|
|
}
|
|
}
|