From c9ce4c480f6a589edfaa13539990f18ff661b28f Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 17:12:31 +0300 Subject: [PATCH 01/30] fix(clickhouse): let the backup sidecar pick up credentials projected after it started With backup.useSystemBucket the clickhouse-backup sidecar took its S3 settings from optional secretKeyRefs on cozy-backups-creds. The platform projects that Secret into the namespace at the first BackupJob, usually after the ClickHouse pod started, and a container's environment is fixed at start, so the sidecar ran without credentials and every backup and restore failed until the pod was restarted by hand. Rotated keys stayed stale the same way. The projector now also writes the s3 section of a clickhouse-backup config into the Secret, and the chart mounts that key, optionally, as the sidecar's config file instead of setting S3 environment variables. clickhouse-backup reloads its config file in each API handler that reaches S3 and kubelet rewrites a mounted Secret in place, so projected and rotated credentials reach the running sidecar. The flat keys stay for the Redis strategy and for ClickHouse releases still rendered by the previous chart. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/credentials_projector.go | 75 +++++++++++---- .../credentials_projector_test.go | 68 +++++++++++++ .../apps/clickhouse/templates/clickhouse.yaml | 95 +++++++++---------- .../apps/clickhouse/tests/backup_test.yaml | 77 ++++++++------- 4 files changed, 211 insertions(+), 104 deletions(-) diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index 4e0c3b75a..eb34d22bc 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "strconv" "strings" "time" @@ -15,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" ) // CredentialsProjectionRequeue is the backoff between reconcile attempts @@ -73,9 +75,8 @@ func IsTransient(err error) bool { // BackupCredentialsConfig describes the platform-managed source Secret // produced by the cozy-backups Bucket, the per-tenant projection target // Secret name, and the bucket coordinates (endpoint used to format the -// FDB blob_credentials account-host key, region carried verbatim for -// drivers like clickhouse-backup that wire it through env, forcePathStyle -// projected as a flat key so the chart-emitted ClickHouse sidecar can read +// FDB blob_credentials account-host key, region carried verbatim, +// forcePathStyle projected so the chart-emitted ClickHouse sidecar can read // a platform value it cannot see directly). Values are wired from // environment variables in main() and the deployment chart populates them // from .Values.backupStorage. @@ -87,9 +88,9 @@ type BackupCredentialsConfig struct { Region string // ForcePathStyle is the string "true"/"false" from // backupStorage.forcePathStyle, projected into the credentials Secret - // under the forcePathStyle key. The app charts cannot read - // backupStorage.* directly, so the ClickHouse sidecar consumes it via - // an optional secretKeyRef as S3_FORCE_PATH_STYLE. + // under the forcePathStyle key and into clickhouse-backup.yml. The app + // charts cannot read backupStorage.* directly, so the ClickHouse + // sidecar takes it from there. ForcePathStyle string } @@ -107,10 +108,11 @@ func (c BackupCredentialsConfig) IsEnabled() bool { // the data so every default Strategy CR can consume the same Secret: // // - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — CNPG, MariaDB, Etcd. -// - accessKey / secretKey + bucketName / endpoint / region — ClickHouse -// sidecar. +// - accessKey / secretKey + bucketName / endpoint / region — Redis +// strategy, and ClickHouse sidecars rendered before clickhouse-backup.yml. // - cloud — Velero credentials file ([default] section). // - blob_credentials.json — FoundationDB backup_agent. +// - clickhouse-backup.yml — ClickHouse sidecar, mounted as its config file. // // The source Secret is read in two formats: // @@ -156,8 +158,8 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr if creds.endpoint == "" { // Scheme-strip the env-var fallback to match the source-Secret path // (parseSourceSecret strips there). The projected `endpoint` key is - // consumed verbatim as S3_ENDPOINT by the ClickHouse sidecar, which - // expects a bare host:port — see https://cozystack.io/docs/v1/operations/services/backup-classes/. + // a bare host:port and each consumer adds its own scheme — see + // https://cozystack.io/docs/v1/operations/services/backup-classes/. creds.endpoint = stripScheme(cfg.Endpoint) } if creds.endpoint == "" { @@ -177,6 +179,13 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr } cloud := buildVeleroCredentialsFile(creds.accessKey, creds.secretKey) + clickHouseConfig, err := buildClickHouseBackupConfig(creds, cfg.ForcePathStyle) + if err != nil { + return &ProjectionError{ + Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("build %s: %v", clickHouseBackupConfigKey, err), + } + } blob, err := buildFDBBlobCredentials(creds.endpoint, creds.accessKey, creds.secretKey) if err != nil { return &ProjectionError{ @@ -233,24 +242,25 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr } target.Data["AWS_ACCESS_KEY_ID"] = []byte(creds.accessKey) target.Data["AWS_SECRET_ACCESS_KEY"] = []byte(creds.secretKey) - // Mirror under the legacy bucket-controller key names so - // chart-emitted sidecars (clickhouse-backup) can consume the - // same Secret without a separate field map. + // Mirror under the legacy bucket-controller key names so the Redis + // strategy and older ClickHouse charts can consume the same Secret + // without a separate field map. target.Data["accessKey"] = []byte(creds.accessKey) target.Data["secretKey"] = []byte(creds.secretKey) target.Data["cloud"] = []byte(cloud) target.Data["blob_credentials.json"] = blob + target.Data[clickHouseBackupConfigKey] = clickHouseConfig // Always overwrite endpoint/bucket/region (delete if empty in // source) so a re-projection cannot leave stale values lingering - // after the bucket-controller rotates the source. Consumers - // (ClickHouse sidecar via secretKeyRef) must see exactly what the - // source carries, not a half-stale Secret. + // after the bucket-controller rotates the source. Consumers must + // see exactly what the source carries, not a half-stale Secret. setOrDelete(target.Data, "endpoint", creds.endpoint) setOrDelete(target.Data, "bucketName", creds.bucket) setOrDelete(target.Data, "region", creds.region) // forcePathStyle comes from the platform config (backupStorage. - // forcePathStyle), not the source Secret: the ClickHouse sidecar - // reads it as S3_FORCE_PATH_STYLE because the app chart cannot see + // forcePathStyle), not the source Secret: ClickHouse sidecars + // rendered before clickhouse-backup.yml read it as + // S3_FORCE_PATH_STYLE because the app chart cannot see // backupStorage.* directly. setOrDelete(target.Data, "forcePathStyle", cfg.ForcePathStyle) return nil @@ -414,6 +424,35 @@ func buildFDBBlobCredentials(endpoint, accessKey, secretKey string) ([]byte, err return json.Marshal(doc) } +// clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup +// config. The ClickHouse chart mounts it as the sidecar's config file +// instead of passing the values as environment: clickhouse-backup reloads +// the file on every API call and kubelet rewrites a mounted Secret in +// place, so credentials projected after the pod started, or rotated since, +// reach a running sidecar. +const clickHouseBackupConfigKey = "clickhouse-backup.yml" + +// buildClickHouseBackupConfig renders that file. The projected endpoint is +// a bare host, and clickhouse-backup's S3 client rejects one without a +// scheme. +func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) ([]byte, error) { + s3 := map[string]any{ + "bucket": creds.bucket, + "endpoint": "https://" + creds.endpoint, + "region": creds.region, + "access_key": creds.accessKey, + "secret_key": creds.secretKey, + } + if forcePathStyle != "" { + pathStyle, err := strconv.ParseBool(forcePathStyle) + if err != nil { + return nil, fmt.Errorf("forcePathStyle %q is not a boolean", forcePathStyle) + } + s3["force_path_style"] = pathStyle + } + return yaml.Marshal(map[string]any{"s3": s3}) +} + // stripScheme returns the endpoint stripped of any leading scheme prefix // (http:// or https://) and preserves the path portion if present. The // resulting form is what `host[:port][/path]` callers (MariaDB strategy, diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index 2b4f0bb1c..c4b154dfa 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" ) func newFakeClient(objs ...client.Object) client.Client { @@ -664,3 +665,70 @@ func TestProject_HappyPath_HTTPEndpoint(t *testing.T) { t.Errorf("blob account host not stripped: %+v", blob.Accounts) } } + +// The ClickHouse sidecar reads its S3 settings from this key as a mounted +// file, which kubelet refreshes in place: clickhouse-backup reloads its +// config file on every API call, while its environment is fixed at start. +func TestProject_WritesClickHouseBackupConfig(t *testing.T) { + cases := []struct { + name string + forcePathStyle string + wantPathStyle *bool + }{ + {name: "path style on", forcePathStyle: "true", wantPathStyle: ptrBool(true)}, + {name: "path style off", forcePathStyle: "false", wantPathStyle: ptrBool(false)}, + {name: "path style unset", forcePathStyle: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + ctx := context.Background() + cfg := defaultCfg() + cfg.ForcePathStyle = tc.forcePathStyle + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v\n%s", clickHouseBackupConfigKey, err, got.Data[clickHouseBackupConfigKey]) + } + for key, want := range map[string]string{ + "bucket": "cozy-backups", + "endpoint": "https://seaweedfs-s3.tenant-root.svc:8333", + "region": "us-east-1", + "access_key": "AK", + "secret_key": "SK", + } { + if doc.S3[key] != want { + t.Errorf("s3.%s = %v, want %q", key, doc.S3[key], want) + } + } + pathStyle, set := doc.S3["force_path_style"] + switch { + case tc.wantPathStyle == nil && set: + t.Errorf("s3.force_path_style = %v, want it left to the clickhouse-backup default", pathStyle) + case tc.wantPathStyle != nil && pathStyle != *tc.wantPathStyle: + t.Errorf("s3.force_path_style = %v, want %v", pathStyle, *tc.wantPathStyle) + } + }) + } +} + +func TestProject_RejectsMalformedForcePathStyle(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + cfg := defaultCfg() + cfg.ForcePathStyle = "yes please" + err := ProjectBackupCredentials(context.Background(), c, cfg, "tenant-acme") + var perr *ProjectionError + if !errors.As(err, &perr) || perr.Reason != ReasonSourceMalformed { + t.Fatalf("err = %v, want a %s ProjectionError", err, ReasonSourceMalformed) + } +} + +func ptrBool(b bool) *bool { return &b } diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 827b5d579..d8791878e 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -225,15 +225,18 @@ spec: - name: REMOTE_STORAGE value: s3 {{- /* + On the useSystemBucket flow every S3 setting, the + platform's backupStorage.forcePathStyle included, comes + from the clickhouse-backup.yml key the platform projects + into cozy-backups-creds, mounted below as the config file. + clickhouse-backup reloads that file on every API call and + kubelet rewrites a mounted Secret in place, so credentials + projected after the Pod started, or rotated since, reach + the running sidecar; environment would be read only at + start and would override the file, so none is set here. + S3_FORCE_PATH_STYLE selects path-style vs vhost-style S3 - addressing: SeaweedFS (the platform default) needs - path-style, while AWS public S3 needs vhost-style in many - regions. On the useSystemBucket flow the value is the - platform's backupStorage.forcePathStyle, which this app - chart cannot read directly, so it is projected into - cozy-backups-creds and consumed via an optional - secretKeyRef (optional: true so the Pod starts before the - first projection). On the legacy paths (own Secret via + addressing. On the legacy paths (own Secret via s3CredentialsSecret.name, or a configured endpoint) we cannot peek inside the Secret at render time, so we keep the historical path-style default — AWS users in that mode @@ -241,12 +244,8 @@ spec: clickhouse-backup config. */}} {{- if .Values.backup.useSystemBucket }} - - name: S3_FORCE_PATH_STYLE - valueFrom: - secretKeyRef: - name: {{ $sidecarSecretName | quote }} - key: forcePathStyle - optional: true + - name: CLICKHOUSE_BACKUP_CONFIG + value: /etc/clickhouse-backup-s3/config.yml {{- else if or (not (eq (default "" .Values.backup.s3CredentialsSecret.name) "")) .Values.backup.endpoint }} - name: S3_FORCE_PATH_STYLE value: "true" @@ -282,44 +281,18 @@ spec: */}} - name: S3_PATH value: {{ $s3Path | quote }} - {{- /* - optional: true in the useSystemBucket path so the Pod - starts even before the platform projector materialises - cozy-backups-creds in this namespace (projection runs at - first BackupJob time; the sidecar surfaces backup errors - only when its HTTP API is exercised). Without this gate - every fresh ClickHouse release with useSystemBucket=true - would hang in CreateContainerConfigError until a - BackupJob fires. Legacy path keeps the strict reference - shape — its Secret is chart-emitted and present at - install time. - */}} - {{- $secretOptional := .Values.backup.useSystemBucket }} + {{- if not .Values.backup.useSystemBucket }} - name: S3_BUCKET - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } - {{- /* - clickhouse-backup's aws-sdk-go-v2 client rejects a scheme-less - S3_ENDPOINT ("was not a valid URI"). On the useSystemBucket flow - the projected cozy-backups-creds.endpoint is a bare host, so - prepend https:// (the platform's CNPG endpoint helper does the - same). Legacy / BYO-Secret endpoints are tenant-supplied with - their own scheme and are consumed verbatim. - */}} - {{- if .Values.backup.useSystemBucket }} - - name: S3_ENDPOINT_HOST - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }}, optional: true } } - - name: S3_ENDPOINT - value: "https://$(S3_ENDPOINT_HOST)" - {{- else }} + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }} } } - name: S3_ENDPOINT valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }} } } - {{- end }} - name: S3_REGION - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }} } } - name: S3_ACCESS_KEY - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }} } } - name: S3_SECRET_KEY - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }} } } + {{- end }} {{- if $endpointCAName }} # Trust the mounted private-CA bundle for the S3 endpoint (see # the `backup.endpointCA` field in values.yaml). SSL_CERT_DIR, @@ -343,6 +316,11 @@ spec: - name: backup-s3-ca mountPath: /etc/ssl/cozy-s3-ca readOnly: true +{{- end }} +{{- if .Values.backup.useSystemBucket }} + - name: backup-s3-config + mountPath: /etc/clickhouse-backup-s3 + readOnly: true {{- end }} securityContext: # clickhouse-server stores data under /var/lib/clickhouse with @@ -366,13 +344,14 @@ spec: runAsGroup: 0 seccompProfile: type: RuntimeDefault -{{- if $endpointCAName }} - # Private-CA bundle for the backup sidecar, projected as ca.crt for - # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds - # the whole ClickHouse Pod in ContainerCreating (naming the Secret in - # `kubectl describe`) rather than mounting an empty trust dir and - # failing later inside clickhouse-backup with an opaque x509 error. +{{- if or $endpointCAName .Values.backup.useSystemBucket }} volumes: +{{- if $endpointCAName }} + # Private-CA bundle for the backup sidecar, projected as ca.crt for + # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds + # the whole ClickHouse Pod in ContainerCreating (naming the Secret in + # `kubectl describe`) rather than mounting an empty trust dir and + # failing later inside clickhouse-backup with an opaque x509 error. - name: backup-s3-ca secret: secretName: {{ $endpointCAName | quote }} @@ -380,6 +359,18 @@ spec: - key: {{ $endpointCAKey | quote }} path: ca.crt {{- end }} +{{- if .Values.backup.useSystemBucket }} + # Optional: the platform projects cozy-backups-creds into this + # namespace at the first BackupJob, after the Pod has started. + - name: backup-s3-config + secret: + secretName: {{ $sidecarSecretName | quote }} + optional: true + items: + - key: clickhouse-backup.yml + path: config.yml +{{- end }} +{{- end }} {{- end }} serviceTemplates: - name: svc-template diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml index 66eda4dc8..7448e02a3 100644 --- a/packages/apps/clickhouse/tests/backup_test.yaml +++ b/packages/apps/clickhouse/tests/backup_test.yaml @@ -2,10 +2,12 @@ suite: backup sidecar S3 wiring (useSystemBucket vs legacy, path-override guard) # Locks in the clickhouse-backup sidecar S3 wiring this PR introduced on the # ClickHouseInstallation PodTemplate: -# - useSystemBucket=true points every S3_* env at the platform-projected -# cozy-backups-creds Secret with optional:true (the Pod starts before the -# first projection) and scopes S3_PATH to /; the -# chart-managed -backup-s3 / -backup Secrets are skipped; +# - useSystemBucket=true mounts the clickhouse-backup.yml key of the +# platform-projected cozy-backups-creds Secret as the sidecar's config +# file with optional:true (the Pod starts before the first projection), +# sets no other S3 setting in the environment, and scopes S3_PATH to +# /; the chart-managed -backup-s3 / +# -backup Secrets are skipped; # - the legacy flow keeps the chart-emitted -backup-s3 Secret and a # strict (non-optional) secretKeyRef; # - s3PathOverride + useSystemBucket fails fast at render time. @@ -25,7 +27,7 @@ set: _cluster: {} tests: - - it: "useSystemBucket=true: sidecar reads cozy-backups-creds (optional) and scopes S3_PATH" + - it: "useSystemBucket=true: sidecar reads its S3 settings from the projected config file and scopes S3_PATH" template: templates/clickhouse.yaml documentSelector: path: kind @@ -45,28 +47,29 @@ tests: - equal: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_PATH")].value value: tenant-test/ch-test + # clickhouse-backup reloads this file on every API call and kubelet + # rewrites the mounted Secret in place, so credentials projected after + # the Pod started, or rotated since, reach the running sidecar. - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name - value: cozy-backups-creds + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="CLICKHOUSE_BACKUP_CONFIG")].value + value: /etc/clickhouse-backup-s3/config.yml - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.optional + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].mountPath + value: /etc/clickhouse-backup-s3 + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].readOnly value: true - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - # The projected endpoint is a bare host; clickhouse-backup needs a URI, so - # the chart prepends https:// via env-var expansion on this flow. - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.key - value: endpoint - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")].value - value: https://$(S3_ENDPOINT_HOST) + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret + value: + secretName: cozy-backups-creds + # The Pod starts before the first projection. + optional: true + items: + - key: clickhouse-backup.yml + path: config.yml - - it: "useSystemBucket=true: S3_FORCE_PATH_STYLE is sourced from cozy-backups-creds, not hardcoded" + - it: "useSystemBucket=true: no S3 setting is fixed in the sidecar environment" template: templates/clickhouse.yaml documentSelector: path: kind @@ -76,18 +79,21 @@ tests: enabled: true useSystemBucket: true asserts: - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.key - value: forcePathStyle - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.optional - value: true + # Environment overrides the config file and is read only at start. - notExists: - # must no longer be the hardcoded literal value: "true" - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].value + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_REGION")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ACCESS_KEY")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")] - it: "useSystemBucket=true: chart-managed backup Secrets are skipped" set: @@ -240,7 +246,10 @@ tests: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="SSL_CERT_DIR")].value value: /etc/ssl/cozy-s3-ca:/etc/ssl/certs - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-ca")].secret.secretName + value: my-s3-ca + - equal: + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret.secretName value: cozy-backups-creds # backup.enabled=false renders no sidecar at all, so the CA must not leak a From f5f50d2a77179c823ad6d6ab1f1e61d9c49a359c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 17:15:10 +0300 Subject: [PATCH 02/30] fix(clickhouse): hold a strategy job until projected credentials reach the sidecar The clickhouse-backup sidecar now reads the projected credentials from a mounted Secret, which kubelet rewrites only after its sync period, about 75 seconds after the change in practice. The first BackupJob projects the Secret and started the strategy Job a second later, so its attempts raced the volume update and the backup could still fail on a sidecar without credentials. The projector records when the projected content last changed, and the ClickHouse driver holds back creating a backup or restore Job until two minutes have passed since, with a WaitingForCredentials condition. An unchanged re-projection keeps the time, so only the first projection and a rotation cause a wait, and a Job already running is never held. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../altinitystrategy_controller.go | 74 +++++++ .../altinitystrategy_controller_test.go | 186 ++++++++++++++++++ .../backupcontroller/credentials_projector.go | 15 ++ .../credentials_projector_test.go | 56 ++++++ 4 files changed, 331 insertions(+) diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index b96d571f0..a99801ac6 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -71,6 +71,56 @@ func validateAltinityApplicationRef(ref corev1.TypedLocalObjectReference) error return nil } +// altinityCredentialsPropagation bounds how long kubelet takes to rewrite +// the mounted cozy-backups-creds in the ClickHouse Pod after it changes: +// its sync period (a minute by default) plus the Secret cache. About 75s was +// measured; the margin covers the rest. +const altinityCredentialsPropagation = 2 * time.Minute + +// credentialsPropagationWait returns how long to hold back creating the +// strategy Job named jobName in namespace, so that the clickhouse-backup +// sidecar, which reads the projected credentials from a mounted file, sees +// their latest content. A Job that already exists, or a namespace without +// projected credentials, does not wait. +func credentialsPropagationWait(ctx context.Context, c client.Client, secretName, namespace, jobName string) (time.Duration, error) { + if secretName == "" { + return 0, nil + } + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: jobName}, &batchv1.Job{}); err == nil { + return 0, nil + } else if !apierrors.IsNotFound(err) { + return 0, err + } + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secretName}, secret); err != nil { + if apierrors.IsNotFound(err) { + return 0, nil + } + return 0, err + } + raw := secret.Annotations[credentialsUpdatedAtAnnotation] + if raw == "" { + return 0, nil + } + updatedAt, err := time.Parse(time.RFC3339, raw) + if err != nil { + return 0, fmt.Errorf("parse %s on Secret %s/%s: %w", credentialsUpdatedAtAnnotation, namespace, secretName, err) + } + return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil +} + +// credentialsWaitCondition is the Ready condition of a job held back by +// credentialsPropagationWait. +func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition { + return metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "WaitingForCredentials", + Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar", + wait.Round(time.Second), secretName), + } +} + // jobNameForBackupJob returns the deterministic batch/v1.Job name used for // the backup pod that drives a BackupJob. Idempotency relies on this being // pure: a retried reconcile must hit the same Job name. @@ -222,6 +272,18 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Altinity strategy: %v", err)) } + wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, j.Namespace, jobNameForBackupJob(j)) + if err != nil { + return ctrl.Result{}, err + } + if wait > 0 { + apimeta.SetStatusCondition(&j.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait)) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: wait}, nil + } + batchJob, err := r.ensureAltinityJob(ctx, j, j.Namespace, jobNameForBackupJob(j), altinityModeBackup, map[string]string{ @@ -623,6 +685,18 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template Altinity strategy: %v", err)) } + wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, targetNamespace, jobNameForRestoreJob(restoreJob)) + if err != nil { + return ctrl.Result{}, err + } + if wait > 0 { + apimeta.SetStatusCondition(&restoreJob.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait)) + if err := r.Status().Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: wait}, nil + } + batchJob, err := r.ensureAltinityRestoreJob(ctx, restoreJob, targetNamespace, jobNameForRestoreJob(restoreJob), altinityModeRestore, map[string]string{ diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 68f18fa28..75564538b 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -3,7 +3,9 @@ package backupcontroller import ( "context" + "errors" "testing" + "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -17,6 +19,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -710,3 +713,186 @@ func TestBackupParameters_RoundTrip(t *testing.T) { t.Errorf("non-parameter key leaked through: %v", got) } } + +func projectedCredentials(namespace string, updatedAt time.Time) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: updatedAt.UTC().Format(time.RFC3339)}, + }, + } +} + +// The ClickHouse sidecar reads the projected credentials from a mounted +// file that kubelet rewrites only some time after the Secret changes, so a +// strategy Job started right after a projection would reach a sidecar with +// no or stale credentials. +func TestReconcileAltinity_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) { + cases := []struct { + name string + secret *corev1.Secret + jobExists bool + wantWait bool + }{ + {name: "credentials changed just now", secret: projectedCredentials("tenant-test", time.Now()), wantWait: true}, + {name: "credentials changed long ago", secret: projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))}, + {name: "no projected credentials", secret: nil}, + {name: "the Job already runs", secret: projectedCredentials("tenant-test", time.Now()), jobExists: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + objs := []client.Object{backupJob, newAltinityStrategy("clickhouse-strategy")} + if tc.secret != nil { + objs = append(objs, tc.secret) + } + if tc.jobExists { + objs = append(objs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}}) + } + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), clientfake.NewClientBuilder().WithObjects(objs...)) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, err := r.reconcileAltinity(ctx, j, resolved) + if err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + ready := meta.FindStatusCondition(j.Status.Conditions, "Ready") + if tc.wantWait { + if len(jobs.Items) != 0 { + t.Errorf("a strategy Job was created %d time(s) before the credentials could reach the sidecar", len(jobs.Items)) + } + if res.RequeueAfter <= 0 || res.RequeueAfter > altinityCredentialsPropagation { + t.Errorf("RequeueAfter = %v, want within (0, %v]", res.RequeueAfter, altinityCredentialsPropagation) + } + if ready == nil || ready.Reason != "WaitingForCredentials" { + t.Errorf("Ready = %+v, want WaitingForCredentials", ready) + } + return + } + if len(jobs.Items) != 1 { + t.Errorf("got %d strategy Jobs, want 1", len(jobs.Items)) + } + if ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("Ready = %+v, the job should not wait", ready) + } + }) + } +} + +func TestReconcileAltinityRestore_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now()))) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + res, err := rr.reconcileAltinityRestore(ctx, rj, backup) + if err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + jobs := &batchv1.JobList{} + if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a restore Job was created before the credentials could reach the sidecar") + } + if res.RequeueAfter <= 0 { + t.Errorf("RequeueAfter = %v, want a wait", res.RequeueAfter) + } + if ready := meta.FindStatusCondition(rj.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCredentials" { + t.Errorf("Ready = %+v, want WaitingForCredentials", ready) + } +} + +// Without credentials projection configured there is no Secret to wait +// for; client-go refuses a Get with an empty name, so it must not be asked. +func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy")). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "" { + return errors.New("resource name may not be empty") + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) + } +} diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index eb34d22bc..99d4e182e 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -1,10 +1,12 @@ package backupcontroller import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "maps" "net/url" "strconv" "strings" @@ -240,6 +242,7 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr if target.Data == nil { target.Data = map[string][]byte{} } + before := maps.Clone(target.Data) target.Data["AWS_ACCESS_KEY_ID"] = []byte(creds.accessKey) target.Data["AWS_SECRET_ACCESS_KEY"] = []byte(creds.secretKey) // Mirror under the legacy bucket-controller key names so the Redis @@ -263,6 +266,12 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr // S3_FORCE_PATH_STYLE because the app chart cannot see // backupStorage.* directly. setOrDelete(target.Data, "forcePathStyle", cfg.ForcePathStyle) + if !maps.EqualFunc(before, target.Data, bytes.Equal) { + if target.Annotations == nil { + target.Annotations = map[string]string{} + } + target.Annotations[credentialsUpdatedAtAnnotation] = time.Now().UTC().Format(time.RFC3339) + } return nil }); err != nil { // If the mutator refused to overwrite an unowned Secret, surface @@ -424,6 +433,12 @@ func buildFDBBlobCredentials(endpoint, accessKey, secretKey string) ([]byte, err return json.Marshal(doc) } +// credentialsUpdatedAtAnnotation records when the projected content last +// changed. A consumer that mounts the Secret as a volume sees a change only +// once kubelet rewrites the volume, so it uses this to tell how fresh its +// copy can be. +const credentialsUpdatedAtAnnotation = "backups.cozystack.io/credentials-updated-at" + // clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup // config. The ClickHouse chart mounts it as the sidecar's config file // instead of passing the values as environment: clickhouse-backup reloads diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index c4b154dfa..208ba253c 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "testing" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -732,3 +733,58 @@ func TestProject_RejectsMalformedForcePathStyle(t *testing.T) { } func ptrBool(b bool) *bool { return &b } + +// Consumers that read the Secret through a mounted volume need to know when +// its content last changed; an unchanged re-projection must keep the time. +func TestProject_StampsWhenContentChanges(t *testing.T) { + src := flatSourceSecret() + c := newFakeClient(src) + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"} + stamp := func() string { + got := &corev1.Secret{} + if err := c.Get(ctx, key, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + return got.Annotations[credentialsUpdatedAtAnnotation] + } + + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("first projection: %v", err) + } + first := stamp() + at, err := time.Parse(time.RFC3339, first) + if err != nil || time.Since(at) > time.Minute { + t.Fatalf("first projection stamp %q is not a recent RFC3339 time: %v", first, err) + } + + old := &corev1.Secret{} + if err := c.Get(ctx, key, old); err != nil { + t.Fatalf("get projected Secret: %v", err) + } + old.Annotations[credentialsUpdatedAtAnnotation] = "2000-01-01T00:00:00Z" + if err := c.Update(ctx, old); err != nil { + t.Fatalf("age the stamp: %v", err) + } + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("unchanged re-projection: %v", err) + } + if got := stamp(); got != "2000-01-01T00:00:00Z" { + t.Fatalf("an unchanged re-projection moved the stamp to %q", got) + } + + rotated := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: src.Namespace, Name: src.Name}, rotated); err != nil { + t.Fatalf("get source Secret: %v", err) + } + rotated.Data["secretKey"] = []byte("SK2") + if err := c.Update(ctx, rotated); err != nil { + t.Fatalf("rotate source Secret: %v", err) + } + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("rotated re-projection: %v", err) + } + if got := stamp(); got == "2000-01-01T00:00:00Z" { + t.Fatalf("a rotation did not move the stamp") + } +} From e7529520471c41053970f708993186f2eeff48ae Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:31:29 +0300 Subject: [PATCH 03/30] test(clickhouse): back up and restore a running release through the system bucket A ClickHouse release on the system bucket starts before the platform projects cozy-backups-creds into its namespace, and no suite exercised that order: the round-trips bring their own Bucket and endpoint, so the sidecar never had to pick up credentials that arrived after it started. The new Test starts the release without the projected Secret and runs a backup and an in-place restore through cozy-default against the bucket as advertised. In CI that host neither resolves nor has a certificate, so the step makes it reachable from tenant-root for its own duration and proves the served certificate before any job starts, which keeps a DNS or TLS problem from reading as a backup failure. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../clickhouse/chainsaw-test.yaml | 176 +++++++++++++++++ .../clickhouse/clickhouse-system-bucket.yaml | 28 +++ .../clickhouse/system-bucket-roundtrip.sh | 185 ++++++++++++++++++ .../clickhouse/system-bucket-tls.yaml | 42 ++++ 4 files changed, 431 insertions(+) create mode 100644 hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml create mode 100755 hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh create mode 100644 hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 939e8f69a..f5b444a84 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -377,3 +377,179 @@ spec: set -eu cd ../../.. NAMESPACE="$NAMESPACE" examples/backups/clickhouse/cleanup.sh +--- +# Backup and in-place restore through the shipped cozy-default BackupClass and +# the platform system bucket, for a ClickHouse whose Pod started before the +# platform projected cozy-backups-creds into its namespace. The sidecar has to +# pick those credentials up while running; until it could, the first BackupJob +# of such a release failed with "no EC2 IMDS role found" and kept failing until +# the Pod was restarted. +# +# Unlike clickhouse-2-backup-roundtrip this targets the system bucket as +# advertised, https://s3.example.org, so system-bucket-roundtrip.sh makes that +# name reachable from tenant-root for the duration of its own step and undoes +# it on exit: see its header for the two shared objects it touches and the +# cert-manager behaviour that relies on. Runs in tenant-root, where the system +# bucket's SeaweedFS and its Ingress controller sit. +# +# The precondition step deletes tenant-root's projected cozy-backups-creds, +# which the mongodb and rabbitmq round-trips leave there after their +# cozy-default BackupJobs. Nothing reads it between suites, and the projector +# writes it again on the next BackupJob, which is the moment under test. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: clickhouse-3-system-bucket-roundtrip +spec: + namespace: tenant-root + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: clickhouse-sysbucket + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + name: e2e-system-bucket-s3 + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + name: seaweedfs-s3-ingress-tls + - describe: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: ingress-seaweedfs-s3 + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: BackupJob + name: sysbucket-backup + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: RestoreJob + name: sysbucket-restore + - podLogs: + selector: app.kubernetes.io/instance=clickhouse-sysbucket + container: clickhouse-backup + tail: 200 + - podLogs: + selector: app=clickhouse-sysbucket-keeper + tail: 100 + steps: + - name: preconditions + try: + - script: + timeout: 2m + content: | + set -eu + if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then + echo "Secret $NAMESPACE/seaweedfs-s3-ingress-tls already exists; this Test would have to replace it" >&2 + exit 1 + fi + if kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF 'BEGIN e2e-clickhouse-system-bucket'; then + echo "kube-system/coredns still carries the e2e-clickhouse-system-bucket block from an earlier run" >&2 + exit 1 + fi + # Fixed names in a persistent namespace: a leftover Succeeded job + # would satisfy the waits in the run step without a new attempt. + kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s + check: + ($error == null): true + + - name: create-certificate-and-probe + try: + - apply: + file: system-bucket-tls.yaml + - assert: + timeout: 2m + resource: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: e2e-system-bucket-s3 + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Pod + metadata: + name: e2e-system-bucket-probe + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: create-clickhouse + try: + - apply: + file: clickhouse-system-bucket.yaml + # 11m: above the 600s install timeout cozystack-api gives this release; + # see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. + - assert: + timeout: 11m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: clickhouse-sysbucket + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: chi-clickhouse-sysbucket-clickhouse-0-0 + status: + readyReplicas: 1 + # The Pod is running with no projected credentials: the state the first + # BackupJob of a fresh release meets. + - error: + timeout: 30s + resource: + apiVersion: v1 + kind: Secret + metadata: + name: cozy-backups-creds + + - name: run + try: + # Inner budgets of system-bucket-roundtrip.sh: + # 240s probe: CoreDNS picking up the rewrite and the Ingress serving + # the Test certificate + # 480s BackupJob to Succeeded, including the driver's 2m hold + # 480s RestoreJob to Succeeded, same hold + # 90s the EXIT trap: Corefile replace plus a 60s Secret delete + # 1290s (21m30s) + # 25m leaves 3m30s for the SQL and kubectl calls between those waits, so + # an inner wait is what fires and names what stalled. + - script: + timeout: 25m + content: | + set -eu + cd ../../.. + NAMESPACE="$NAMESPACE" hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh + check: + ($error == null): true + finally: + # Chainsaw did not apply these: the jobs come from the script and the + # Backup from the controller, and deleting a BackupJob reclaims no Backup. + # The Certificate's Secret outlives the Certificate Chainsaw deletes. + - description: system-bucket round-trip cleanup + script: + timeout: 3m + content: | + set -eu + backup=$(kubectl -n "$NAMESPACE" get backupjob sysbucket-backup --ignore-not-found -o jsonpath='{.status.backupRef.name}') + kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + if [ -n "$backup" ]; then + kubectl -n "$NAMESPACE" delete backup "$backup" --ignore-not-found --wait=true --timeout=60s + fi + kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml new file mode 100644 index 000000000..f7327b46d --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml @@ -0,0 +1,28 @@ +# ClickHouse on the platform system bucket for the +# clickhouse-3-system-bucket-roundtrip Test. No S3 coordinates are given: the +# sidecar reads them from the clickhouse-backup.yml key the platform projects +# into cozy-backups-creds at the first BackupJob, after this Pod has started. +# backup.endpointCA trusts the SeaweedFS CA that signs the certificate the Test +# serves for s3.example.org. +apiVersion: apps.cozystack.io/v1alpha1 +kind: ClickHouse +metadata: + name: sysbucket +spec: + size: 5Gi + logStorageSize: 1Gi + shards: 1 + replicas: 1 + resourcesPreset: nano + resources: {} + backup: + enabled: true + useSystemBucket: true + endpointCA: + name: e2e-system-bucket-s3-tls + key: ca.crt + clickhouseKeeper: + enabled: true + replicas: 1 + size: 1Gi + resourcesPreset: micro diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh new file mode 100755 index 000000000..4ca52024b --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# Backup and in-place restore of the `sysbucket` ClickHouse through the +# cozy-default BackupClass, against the platform system bucket. Run from the +# repository root by the clickhouse-3-system-bucket-roundtrip Test, which has +# already started the release with no cozy-backups-creds in its namespace. +# +# The system bucket is addressed as https://s3.example.org, the S3 Ingress host +# the COSI driver advertises. In CI that name does not resolve and its ACME +# certificate is never issued, so for the duration of this script, and only of +# this script, two shared objects are changed and the EXIT trap below puts both +# back: +# +# - the S3 Ingress TLS Secret seaweedfs-s3-ingress-tls is created from the +# Test's own certificate (SeaweedFS CA, s3.example.org). It relies on +# cert-manager leaving a Secret's data alone while the Certificate that +# names it has no completed issuance, verified with cert-manager v1.20.2: +# a fresh ACME order would replace this certificate, and ingress-nginx +# would then serve one the sidecar does not trust, which fails the probe +# below rather than the backup. +# - the host CoreDNS Corefile gets one marked `rewrite name exact` line for +# s3.example.org. The host CoreDNS is Talos-managed and imports no snippet +# files, so the block goes into the live Corefile the way ouroboros writes +# its own; ouroboros is off in CI and no other suite uses this name. +set -euo pipefail + +source examples/backups/clickhouse/00-helpers.sh + +: "${NAMESPACE:?}" +CLICKHOUSE_NAME=sysbucket +BACKUPJOB_NAME=sysbucket-backup +RESTOREJOB_NAME=sysbucket-restore +S3_HOST=s3.example.org +INGRESS_SECRET=seaweedfs-s3-ingress-tls +CERT_SECRET=e2e-system-bucket-s3-tls +PROBE_POD=e2e-system-bucket-probe +OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket +DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ===' +DNS_END=' # === END e2e-clickhouse-system-bucket ===' +DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local" +# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's +# reload plugin polls every 30s; the rest is the 30s answer cache. +PROBE_TIMEOUT=240 +# Each covers the driver's 2m hold after the credentials change plus the +# strategy Job itself. +JOB_TIMEOUT=480 + +workdir=$(mktemp -d) + +corefile_has_block() { + kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN" +} + +remove_dns_block() { + kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json" + local begins + begins=$(jq -r '.data.Corefile' "$workdir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true) + if [ "$begins" = 0 ]; then + return 0 + fi + if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$workdir/coredns.json" | grep -qF -- "$DNS_END"; then + log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand" + return 1 + fi + jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \ + '.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \ + "$workdir/coredns.json" >"$workdir/coredns-clean.json" + kubectl replace -f "$workdir/coredns-clean.json" >/dev/null + log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns" +} + +restore_shared_state() { + local rc=$? + set +e + remove_dns_block || rc=1 + if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then + kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1 + fi + rm -rf "$workdir" + exit "$rc" +} + +if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -o name >/dev/null 2>&1; then + log_error "Secret ${NAMESPACE}/${INGRESS_SECRET} already exists; this Test serves its own certificate under that name and will not overwrite one it did not create" + exit 1 +fi +if corefile_has_block; then + log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand" + exit 1 +fi +trap restore_shared_state EXIT + +log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress" +kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \ + | jq --arg name "$INGRESS_SECRET" '{apiVersion, kind, type, + metadata: {name: $name, labels: {"cozystack.io/e2e-owner": "clickhouse-system-bucket"}}, + data: {"tls.crt": .data["tls.crt"], "tls.key": .data["tls.key"], "ca.crt": .data["ca.crt"]}}' \ + | kubectl -n "$NAMESPACE" create -f - + +log_step "Rewriting ${S3_HOST} to ${DNS_TARGET} in the host CoreDNS" +kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json" +jq -e '.data.Corefile | startswith(".:53 {\n")' "$workdir/coredns.json" >/dev/null \ + || { log_error "kube-system/coredns Corefile does not open with '.:53 {'; refusing to edit it"; exit 1; } +# The resourceVersion read above travels with the replace, so a concurrent +# writer makes it fail instead of being overwritten. +jq --arg block "${DNS_BEGIN}"$'\n'" rewrite name exact ${S3_HOST} ${DNS_TARGET}"$'\n'"${DNS_END}" \ + '.data.Corefile |= sub("^\\.:53 \\{\n"; ".:53 {\n" + $block + "\n")' \ + "$workdir/coredns.json" >"$workdir/coredns-new.json" +kubectl replace -f "$workdir/coredns-new.json" >/dev/null + +log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAMESPACE}" +kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt" +want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \ + | openssl x509 -noout -fingerprint -sha256) +elapsed=0 +while true; do + # --cacert makes curl verify both the chain and the host name; %{certs} + # prints the chain it was served, leaf first. + if served=$(kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ + "cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \ + <"$workdir/ca.crt" 2>"$workdir/probe.err"); then + got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \ + | openssl x509 -noout -fingerprint -sha256) + if [ "$got" = "$want" ]; then + log_success "${S3_HOST} resolves in ${NAMESPACE} and serves the Test certificate" + break + fi + echo "served ${got}, want ${want}" >"$workdir/probe.err" + fi + if [ "$elapsed" -ge "$PROBE_TIMEOUT" ]; then + log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")" + kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true + kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true + exit 1 + fi + sleep 10 + elapsed=$((elapsed + 10)) +done + +log_step "Writing the sentinel row" +clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" +clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')" + +log_step "Backing up through cozy-default" +kubectl apply -f - <= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; } +log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored" diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml new file mode 100644 index 000000000..06203a299 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml @@ -0,0 +1,42 @@ +# Certificate and probe Pod for the clickhouse-3-system-bucket-roundtrip Test. +# +# The certificate is signed by the SeaweedFS CA already present in tenant-root, +# so its Secret carries that CA as ca.crt: the ClickHouse release trusts it via +# backup.endpointCA, and the round-trip script copies the same key pair under +# the name the platform's S3 Ingress serves for s3.example.org. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: e2e-system-bucket-s3 +spec: + secretName: e2e-system-bucket-s3-tls + dnsNames: + - s3.example.org + issuerRef: + kind: Issuer + name: seaweedfs-ca-issuer +--- +# Resolves and fetches s3.example.org from inside tenant-root, where the +# ClickHouse sidecar runs, so the script can prove the DNS rewrite and the +# served certificate before it spends minutes on a BackupJob. The image is the +# one hack/e2e-install-cozystack.bats pre-pulls; it carries curl but no openssl. +apiVersion: v1 +kind: Pod +metadata: + name: e2e-system-bucket-probe +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 0 + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: probe + image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47 + command: [sleep, "3600"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] From 2e37aaec5f46a88c01f65764788ab8e58ab7ac7d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:31:59 +0300 Subject: [PATCH 04/30] docs(backups): drop the claim that a region change needs a ClickHouse restart The ClickHouse sidecar on the system bucket no longer reads S3 settings from environment set at start, so a re-projected region reaches it without a Pod rollout, and the contracts fixture no longer depends on optional Secret keys to start its Pod. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml | 4 ++-- packages/system/backupstrategy-controller/values.yaml | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml index b88e7dd6d..fb0979146 100644 --- a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml @@ -8,8 +8,8 @@ # backup.enabled=true with placeholder S3 coordinates is deliberate: the # sidecar runs `clickhouse-backup server`, which starts its HTTP API without # validating S3 (ALLOW_EMPTY_BACKUPS=true, lazy connect — the same reason the -# platform useSystemBucket flow marks the S3 secret keys optional so the Pod -# starts before any bucket is projected). So the Pod becomes Ready and the +# platform useSystemBucket flow can start the Pod before any bucket is +# projected). So the Pod becomes Ready and the # chart<->strategy contract is assertable with no reachable object store. The # real S3 backup/restore round-trip lives in clickhouse-2-backup-roundtrip, # which runs in tenant-root against the in-cluster SeaweedFS. diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 6cf39c65a..2a4152597 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -105,10 +105,6 @@ backupStorage: endpoint: http://seaweedfs-s3.tenant-root.svc.cozy.local:8333 # region is re-projected into cozy-backups-creds on the next reconcile # (1-minute tick for system namespaces, on-demand for tenant namespaces). - # However, charts that consume the region via secretKeyRef in env vars — - # the chart-emitted ClickHouse sidecar on `useSystemBucket: true` is the - # only such consumer today — only pick up the new value on Pod restart. - # Plan a rollout of those Pods when you change this value. region: us-east-1 forcePathStyle: true # systemSecretName — the human-friendly Secret produced by the Bucket From 38b00730ff072b6b03360e8ae12b047d7568c4ac Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:40:01 +0300 Subject: [PATCH 05/30] fix(clickhouse): keep the clickhouse-backup default region when none is configured The environment the sidecar used to read had no S3_REGION when the platform carried no region, which left clickhouse-backup on its own us-east-1. Writing an empty region into the config file replaces that default with an empty one, and requests are then signed for no region. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/credentials_projector.go | 6 ++++- .../credentials_projector_test.go | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index 99d4e182e..ddf5ff8ef 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -454,10 +454,14 @@ func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) ([]by s3 := map[string]any{ "bucket": creds.bucket, "endpoint": "https://" + creds.endpoint, - "region": creds.region, "access_key": creds.accessKey, "secret_key": creds.secretKey, } + // An empty region in the file would override clickhouse-backup's own + // us-east-1 default instead of leaving it in place. + if creds.region != "" { + s3["region"] = creds.region + } if forcePathStyle != "" { pathStyle, err := strconv.ParseBool(forcePathStyle) if err != nil { diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index 208ba253c..78ffb6c94 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -721,6 +721,31 @@ func TestProject_WritesClickHouseBackupConfig(t *testing.T) { } } +// clickhouse-backup defaults the S3 region to us-east-1; an empty value in +// its config file would replace that default rather than fall back to it. +func TestProject_ClickHouseBackupConfigOmitsAnEmptyRegion(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + ctx := context.Background() + cfg := defaultCfg() + cfg.Region = "" + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err) + } + if region, set := doc.S3["region"]; set { + t.Errorf("s3.region = %q, want it left to the clickhouse-backup default", region) + } +} + func TestProject_RejectsMalformedForcePathStyle(t *testing.T) { c := newFakeClient(flatSourceSecret()) cfg := defaultCfg() From 5f636ce62469aa313a3f1a67cba4e72dbf8f4d87 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:43:14 +0300 Subject: [PATCH 06/30] fix(clickhouse): do not hold strategy jobs on a credentials stamp that cannot expire The projector rewrites the credentials-updated-at stamp only when the projected content changes, so a stamp the hold cannot use stays put. An unreadable one failed every reconcile with no condition on the job, and one far in the future held each job again on every reconcile; in both cases every ClickHouse backup and restore in the namespace stopped for good. Such a stamp is now logged and ignored. A stamp only slightly ahead, as a skewed clock writes it, still waits, and that wait ends. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../altinitystrategy_controller.go | 10 ++++- .../altinitystrategy_controller_test.go | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index a99801ac6..c3c609350 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -102,9 +102,17 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName if raw == "" { return 0, nil } + // The projector rewrites the stamp only when the content changes, so a + // stamp this hold cannot use stays as it is: holding on it would block + // every Job in the namespace for good. updatedAt, err := time.Parse(time.RFC3339, raw) if err != nil { - return 0, fmt.Errorf("parse %s on Secret %s/%s: %w", credentialsUpdatedAtAnnotation, namespace, secretName, err) + getLogger(ctx).Error(err, "ignoring unreadable credentials stamp", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation) + return 0, nil + } + if updatedAt.After(time.Now().Add(altinityCredentialsPropagation)) { + getLogger(ctx).Error(nil, "ignoring a credentials stamp further ahead than clock skew explains", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation, "value", raw) + return 0, nil } return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil } diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 75564538b..717b425d1 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -896,3 +896,44 @@ func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) { t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) } } + +func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp}, + }, + } +} + +// The stamp is only ever rewritten when the projected content changes, so a +// value the hold cannot use must not hold the job: an unreadable one would +// fail every reconcile, and one far in the future would wait again on each. +// A stamp a little ahead, as a skewed clock writes it, still waits, but only +// until that stamp plus the propagation time. +func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) { + cases := []struct { + name string + stamp string + wantMin time.Duration + wantMax time.Duration + }{ + {name: "unreadable", stamp: "garbage"}, + {name: "far in the future", stamp: "3000-01-01T00:00:00Z"}, + {name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339), + wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build() + wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job") + if err != nil { + t.Fatalf("credentialsPropagationWait: %v", err) + } + if wait < tc.wantMin || wait > tc.wantMax { + t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax) + } + }) + } +} From 6345b939d2eef42156d2bbcafd59e859be307326 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:43:15 +0300 Subject: [PATCH 07/30] fix(clickhouse): stop reporting a credentials wait once the strategy job runs The hold sets Ready=False/WaitingForCredentials with a countdown, and nothing removed it once the Job was created, so a backup or restore that ran for minutes kept saying it was still waiting. The condition is dropped when the Job runs; a Ready condition set for another reason is left alone. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../altinitystrategy_controller.go | 24 +++- .../altinitystrategy_controller_test.go | 104 ++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index c3c609350..c72e30719 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -117,13 +117,25 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil } +// clearCredentialsWait drops the Ready condition credentialsWaitCondition +// left behind, once the job no longer waits. It reports whether it did. +func clearCredentialsWait(conditions *[]metav1.Condition) bool { + ready := apimeta.FindStatusCondition(*conditions, "Ready") + if ready == nil || ready.Reason != credentialsWaitReason { + return false + } + return apimeta.RemoveStatusCondition(conditions, "Ready") +} + +const credentialsWaitReason = "WaitingForCredentials" + // credentialsWaitCondition is the Ready condition of a job held back by // credentialsPropagationWait. func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition { return metav1.Condition{ Type: "Ready", Status: metav1.ConditionFalse, - Reason: "WaitingForCredentials", + Reason: credentialsWaitReason, Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar", wait.Round(time.Second), secretName), } @@ -336,6 +348,11 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return r.markBackupJobFailed(ctx, j, message) default: + if clearCredentialsWait(&j.Status.Conditions) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: altinityPollInterval}, nil } } @@ -741,6 +758,11 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res return r.markRestoreJobFailed(ctx, restoreJob, message) default: + if clearCredentialsWait(&restoreJob.Status.Conditions) { + if err := r.Status().Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: altinityPollInterval}, nil } } diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 717b425d1..e521d2d46 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -907,6 +907,100 @@ func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret { } } +func waitingForCredentials() []metav1.Condition { + return []metav1.Condition{credentialsWaitCondition("cozy-backups-creds", time.Minute)} +} + +// Once the hold is over and the Job runs, a Ready condition still saying the +// job is waiting for credentials describes a state the job has left. +func TestReconcileAltinity_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Conditions: waitingForCredentials()}, + } + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"), + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithStatusSubresource(&backupsv1alpha1.BackupJob{})) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) + } + stored := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), stored); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("stored Ready = %+v while the Job runs", ready) + } +} + +func TestReconcileAltinityRestore_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning, + Conditions: waitingForCredentials()}, + } + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithStatusSubresource(&backupsv1alpha1.RestoreJob{})) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + stored := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), stored); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("stored Ready = %+v while the Job runs", ready) + } +} + // The stamp is only ever rewritten when the projected content changes, so a // value the hold cannot use must not hold the job: an unreadable one would // fail every reconcile, and one far in the future would wait again on each. @@ -937,3 +1031,13 @@ func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) }) } } + +func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) { + conditions := []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "StrategyNotReady"}} + if clearCredentialsWait(&conditions) { + t.Errorf("clearCredentialsWait reported clearing a Ready condition with reason StrategyNotReady") + } + if ready := meta.FindStatusCondition(conditions, "Ready"); ready == nil || ready.Reason != "StrategyNotReady" { + t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready) + } +} From dbcc2af6791c320a60b80662e78074bcd23b87cf Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:43:55 +0300 Subject: [PATCH 08/30] test(clickhouse): require the system-bucket round-trip to run on the sidecar that started without credentials A Pod restart during the round-trip would hand the sidecar fresh credentials at start, which is how the old environment-based sidecar got them, and the Test would pass without the running sidecar ever picking them up. The Pod uid and the sidecar's restart count must stay the same across both jobs. The probe deadline is now measured on the clock: counting only its pauses let a hanging exec stretch it past the step budget, so the step timeout would fire before the probe could say what stalled. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../clickhouse/chainsaw-test.yaml | 9 ++++---- .../clickhouse/system-bucket-roundtrip.sh | 21 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index f5b444a84..d96013140 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -521,13 +521,14 @@ spec: - name: run try: # Inner budgets of system-bucket-roundtrip.sh: - # 240s probe: CoreDNS picking up the rewrite and the Ingress serving - # the Test certificate + # 280s probe: a 240s deadline for CoreDNS picking up the rewrite and + # the Ingress serving the Test certificate, plus one last 30s + # attempt and its 10s pause # 480s BackupJob to Succeeded, including the driver's 2m hold # 480s RestoreJob to Succeeded, same hold # 90s the EXIT trap: Corefile replace plus a 60s Secret delete - # 1290s (21m30s) - # 25m leaves 3m30s for the SQL and kubectl calls between those waits, so + # 1330s (22m10s) + # 25m leaves 2m50s for the SQL and kubectl calls between those waits, so # an inner wait is what fires and names what stalled. - script: timeout: 25m diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh index 4ca52024b..fe1361418 100755 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -111,11 +111,11 @@ log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAM kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt" want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \ | openssl x509 -noout -fingerprint -sha256) -elapsed=0 +deadline=$((SECONDS + PROBE_TIMEOUT)) while true; do # --cacert makes curl verify both the chain and the host name; %{certs} # prints the chain it was served, leaf first. - if served=$(kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ + if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ "cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \ <"$workdir/ca.crt" 2>"$workdir/probe.err"); then got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \ @@ -126,16 +126,26 @@ while true; do fi echo "served ${got}, want ${want}" >"$workdir/probe.err" fi - if [ "$elapsed" -ge "$PROBE_TIMEOUT" ]; then + if [ "$SECONDS" -ge "$deadline" ]; then log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")" kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true exit 1 fi sleep 10 - elapsed=$((elapsed + 10)) done +# A restarted Pod reads its credentials afresh at start, which is what the +# old environment-based sidecar needed; the round-trip proves the fix only if +# the sidecar that started without them is the one that serves both jobs. +sidecar_identity() { + kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \ + -o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}' +} +sidecar_before=$(sidecar_identity) +[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \ + || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; } + log_step "Writing the sentinel row" clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')" @@ -182,4 +192,7 @@ wait_for_field restorejob "$RESTOREJOB_NAME" '{.status.phase}' Succeeded "$NAMES count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel" | tr -d '[:space:]') [[ "$count" =~ ^[0-9]+$ ]] || { log_error "non-numeric sentinel count: '${count}'"; exit 1; } (( count >= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; } +sidecar_after=$(sidecar_identity) +[ "$sidecar_after" = "$sidecar_before" ] \ + || { log_error "the ClickHouse Pod or its clickhouse-backup container restarted during the round-trip (uid restarts: '${sidecar_before}' then '${sidecar_after}')"; exit 1; } log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored" From 31b905a1ce938b84db90deaadde1cf91796f4387 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:50:35 +0300 Subject: [PATCH 09/30] docs(clickhouse): note that the backup path stays in the sidecar environment The chart comment said every S3 setting of the system-bucket flow comes from the mounted config file, but S3_PATH is still set in the environment: the platform's file carries no path, and the prefix belongs to the release rather than to the platform. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- packages/apps/clickhouse/templates/clickhouse.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index d8791878e..4934e8db1 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -225,15 +225,17 @@ spec: - name: REMOTE_STORAGE value: s3 {{- /* - On the useSystemBucket flow every S3 setting, the - platform's backupStorage.forcePathStyle included, comes + On the useSystemBucket flow every S3 setting but the + release's own S3_PATH below, the platform's + backupStorage.forcePathStyle included, comes from the clickhouse-backup.yml key the platform projects into cozy-backups-creds, mounted below as the config file. clickhouse-backup reloads that file on every API call and kubelet rewrites a mounted Secret in place, so credentials projected after the Pod started, or rotated since, reach the running sidecar; environment would be read only at - start and would override the file, so none is set here. + start and would override the file, so none of those is + set here. S3_FORCE_PATH_STYLE selects path-style vs vhost-style S3 addressing. On the legacy paths (own Secret via From 0ad642e68df217c26e1abfe333adc9edb730388a Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:05:31 +0300 Subject: [PATCH 10/30] test(clickhouse): size the system-bucket round-trip server for SQL The round-trip ClickHouse ran on the nano preset, whose 128Mi limit the clickhouse-1 contract suite never exceeds because it sends no query. This suite writes a sentinel row before the backup, and the server was OOM-killed on that first statement (lastState OOMKilled, exit 137), so the Test failed before it reached the behaviour under test. clickhouse-2 runs the same statements on the small preset. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml index f7327b46d..0ed1dd4a4 100644 --- a/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml @@ -13,7 +13,9 @@ spec: logStorageSize: 1Gi shards: 1 replicas: 1 - resourcesPreset: nano + # The round-trip runs SQL against this server; at nano's 128Mi the first + # query OOM-kills it, so it takes the preset clickhouse-2 runs SQL on. + resourcesPreset: small resources: {} backup: enabled: true From 003bc3ed0bf1c725c0f1587881f6bf59c41ec398 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:06:53 +0300 Subject: [PATCH 11/30] test(clickhouse): fail the round-trip when CoreDNS cannot be restored The round-trip puts the host CoreDNS Corefile back from its EXIT trap, which runs with errexit off. A refused kubectl replace, for instance a resourceVersion conflict with another writer, fell through to the success message and returned 0, so the run stayed green while the rewrite remained in a cluster-wide object and the next run refused to start. The restore now returns the failure. The restore moves into a file of its own so a unit test can run it against a stubbed kubectl and so other callers can share it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- ...clickhouse-system-bucket-shared-state.bats | 75 +++++++++++++++++++ .../clickhouse/system-bucket-roundtrip.sh | 37 +-------- .../clickhouse/system-bucket-shared-state.sh | 53 +++++++++++++ 3 files changed, 132 insertions(+), 33 deletions(-) create mode 100644 hack/clickhouse-system-bucket-shared-state.bats create mode 100644 hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats new file mode 100644 index 000000000..cc0bbf633 --- /dev/null +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats +# The clickhouse-3-system-bucket-roundtrip Test edits the host CoreDNS Corefile +# and must put it back. Its restore runs from an EXIT trap under `set +e`, so a +# restore that fails without saying so leaves the rewrite in a cluster-wide +# object and still reports a green run; the next run then refuses to start. +# These run the restore against a stubbed kubectl. + +load test_helper + +LIB=hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh +HELPERS=examples/backups/clickhouse/00-helpers.sh + +setup() { + strict_setup + STUB=$(mktemp -d) + # A Corefile carrying the Test's marked block, as the run step leaves it. + jq -n '{apiVersion: "v1", kind: "ConfigMap", + metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"}, + data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \ + >"$STUB/coredns.json" + cat >"$STUB/kubectl" <<'EOF' +#!/bin/bash +case "$*" in + *"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;; + replace*) + cp "${@: -1}" "$STUB/replaced.json" + if [ -n "${REPLACE_FAILS:-}" ]; then + echo 'Error from server (Conflict): Operation cannot be fulfilled on configmaps "coredns": the object has been modified' >&2 + exit 1 + fi + ;; + *"get secret"*) ;; + *) echo "unexpected kubectl $*" >&2; exit 2 ;; +esac +EOF + chmod +x "$STUB/kubectl" + export STUB +} + +restore() { + PATH="$STUB:$PATH" NAMESPACE=tenant-root bash -c ' + set -euo pipefail + source "$1" + source "$2" + set +e + restore_shared_state + ' _ "$HELPERS" "$LIB" +} + +@test "a CoreDNS restore the API server refuses fails the restore" { + export REPLACE_FAILS=1 + err=$(mktemp) + if restore 2>"$err"; then + echo "restore_shared_state returned 0 although kubectl replace was refused" >&2 + cat "$err" >&2 + false + fi + if grep -q 'Removed the s3.example.org rewrite' "$err"; then + echo "restore_shared_state reported the rewrite removed although kubectl replace was refused:" >&2 + cat "$err" >&2 + false + fi +} + +@test "a CoreDNS restore sends the Corefile back without the marked block" { + restore + jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null || { + echo "restored Corefile:" >&2 + jq -r '.data.Corefile' "$STUB/replaced.json" >&2 + false + } + # The resourceVersion read travels with the replace, so a concurrent + # writer makes it fail rather than be overwritten. + [ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ] +} diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh index fe1361418..88a3cf2b5 100755 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -24,18 +24,14 @@ set -euo pipefail source examples/backups/clickhouse/00-helpers.sh +source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh : "${NAMESPACE:?}" CLICKHOUSE_NAME=sysbucket BACKUPJOB_NAME=sysbucket-backup RESTOREJOB_NAME=sysbucket-restore -S3_HOST=s3.example.org -INGRESS_SECRET=seaweedfs-s3-ingress-tls CERT_SECRET=e2e-system-bucket-s3-tls PROBE_POD=e2e-system-bucket-probe -OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket -DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ===' -DNS_END=' # === END e2e-clickhouse-system-bucket ===' DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local" # Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's # reload plugin polls every 30s; the rest is the 30s answer cache. @@ -46,35 +42,10 @@ JOB_TIMEOUT=480 workdir=$(mktemp -d) -corefile_has_block() { - kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN" -} - -remove_dns_block() { - kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json" - local begins - begins=$(jq -r '.data.Corefile' "$workdir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true) - if [ "$begins" = 0 ]; then - return 0 - fi - if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$workdir/coredns.json" | grep -qF -- "$DNS_END"; then - log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand" - return 1 - fi - jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \ - '.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \ - "$workdir/coredns.json" >"$workdir/coredns-clean.json" - kubectl replace -f "$workdir/coredns-clean.json" >/dev/null - log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns" -} - -restore_shared_state() { +on_exit() { local rc=$? set +e - remove_dns_block || rc=1 - if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then - kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1 - fi + restore_shared_state || rc=1 rm -rf "$workdir" exit "$rc" } @@ -87,7 +58,7 @@ if corefile_has_block; then log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand" exit 1 fi -trap restore_shared_state EXIT +trap on_exit EXIT log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress" kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \ diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh new file mode 100644 index 000000000..f121eaeee --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# The two shared objects system-bucket-roundtrip.sh changes for the duration +# of its run, and how to put them back. Sourced by that script, whose EXIT trap +# restores them. Needs NAMESPACE and log_success/log_error from +# examples/backups/clickhouse/00-helpers.sh. + +S3_HOST=s3.example.org +INGRESS_SECRET=seaweedfs-s3-ingress-tls +OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket +DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ===' +DNS_END=' # === END e2e-clickhouse-system-bucket ===' + +corefile_has_block() { + kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN" +} + +remove_dns_block() { + local dir + dir=$(mktemp -d) + kubectl -n kube-system get configmap coredns -o json >"$dir/coredns.json" || { rm -rf "$dir"; return 1; } + local begins + begins=$(jq -r '.data.Corefile' "$dir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true) + if [ "$begins" = 0 ]; then + rm -rf "$dir" + return 0 + fi + if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$dir/coredns.json" | grep -qF -- "$DNS_END"; then + log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand" + rm -rf "$dir" + return 1 + fi + jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \ + '.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \ + "$dir/coredns.json" >"$dir/coredns-clean.json" + # Callers run this with errexit off, so a refused replace has to return. + if ! kubectl replace -f "$dir/coredns-clean.json" >/dev/null; then + log_error "could not remove the ${S3_HOST} rewrite from kube-system/coredns; remove the marked block by hand" + rm -rf "$dir" + return 1 + fi + log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns" + rm -rf "$dir" +} + +# Returns non-zero when either object could not be put back. +restore_shared_state() { + local rc=0 + remove_dns_block || rc=1 + if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then + kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1 + fi + return "$rc" +} From a66219899143042c5b6a9b8809d6892f98821a33 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:07:51 +0300 Subject: [PATCH 12/30] test(clickhouse): restore the round-trip's shared objects after a step ceiling The round-trip changes the host CoreDNS Corefile and the S3 Ingress TLS Secret and puts both back from its EXIT trap. When the run step's ceiling fires, Chainsaw kills the shell and the trap never runs, so the rewrite stays in a cluster-wide object and the next run refuses to start. The step's finally runs regardless, so it repeats the restore; after a run whose trap already restored them it finds nothing to undo. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../clickhouse-system-bucket-shared-state.bats | 18 ++++++++++++++++++ .../e2e-chainsaw/clickhouse/chainsaw-test.yaml | 15 +++++++++++++++ .../clickhouse/system-bucket-shared-state.sh | 5 +++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index cc0bbf633..2f7a79197 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -73,3 +73,21 @@ restore() { # writer makes it fail rather than be overwritten. [ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ] } + +@test "the run step's finally restores the shared objects" { + # When the step ceiling fires, Chainsaw kills the shell and the EXIT trap + # of system-bucket-roundtrip.sh never runs; the step's finally still does. + # Chainsaw runs a script's content with sh from the suite's directory. + suite=hack/e2e-chainsaw/clickhouse + finally=$(yq 'select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") + | .spec.steps[] | select(.name == "run") | .finally[] + | select(.description == "restore the host CoreDNS and the S3 Ingress TLS Secret") | .script.content' \ + "$suite/chainsaw-test.yaml") + [ -n "$finally" ] || { echo "the run step has no finally restoring the shared objects" >&2; false; } + (cd "$suite" && PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$finally") + [ -f "$STUB/replaced.json" ] || { + echo "the run step's finally did not put the Corefile back" >&2 + false + } + jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null +} diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index d96013140..03130cfb3 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -539,6 +539,21 @@ spec: check: ($error == null): true finally: + # The script's EXIT trap puts the Corefile and the Ingress TLS Secret + # back, but a step ceiling kills the shell before the trap runs. A no-op + # when the trap already did it. + - description: restore the host CoreDNS and the S3 Ingress TLS Secret + script: + timeout: 2m + content: | + set -eu + cd ../../.. + # Chainsaw runs this under sh; the helpers are bash. + NAMESPACE="$NAMESPACE" bash -euo pipefail -c ' + source examples/backups/clickhouse/00-helpers.sh + source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh + restore_shared_state + ' # Chainsaw did not apply these: the jobs come from the script and the # Backup from the controller, and deleting a BackupJob reclaims no Backup. # The Certificate's Secret outlives the Certificate Chainsaw deletes. diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh index f121eaeee..8275e8c13 100644 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh @@ -1,8 +1,9 @@ #!/bin/bash # The two shared objects system-bucket-roundtrip.sh changes for the duration # of its run, and how to put them back. Sourced by that script, whose EXIT trap -# restores them. Needs NAMESPACE and log_success/log_error from -# examples/backups/clickhouse/00-helpers.sh. +# restores them, and by the finally of its Chainsaw step, which restores them +# when the step ceiling killed the script before the trap could run. Needs +# NAMESPACE and log_success/log_error from examples/backups/clickhouse/00-helpers.sh. S3_HOST=s3.example.org INGRESS_SECRET=seaweedfs-s3-ingress-tls From cabe9a07e28d14089bccb0c072ec338122da23e2 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:08:50 +0300 Subject: [PATCH 13/30] test(clickhouse): reclaim the round-trip Backup by its name The Altinity driver names the Backup after the BackupJob and adopts one that already exists under that name. The round-trip's cleanup deleted the Backup only through the BackupJob's status.backupRef, so a Backup whose BackupJob never recorded it survived, and the next run's BackupJob would have adopted it in place of a fresh backup. Both the cleanup and the preconditions now delete it by name; the preconditions ceiling grows to cover the extra wait. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- ...clickhouse-system-bucket-shared-state.bats | 32 +++++++++++++++++++ .../clickhouse/chainsaw-test.yaml | 13 ++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index 2f7a79197..e76cc0a1d 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -91,3 +91,35 @@ restore() { } jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null } + +# The Altinity driver names the Backup after the BackupJob and adopts one that +# already exists under that name, so a Backup left behind would stand in for +# the next run's backup. It is reclaimed by name, whether or not the BackupJob +# recorded it in status.backupRef. +@test "the round-trip reclaims its Backup by name before and after the run" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + # Nothing is left over from an earlier run, and no BackupJob recorded a + # backupRef. + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + for selector in \ + 'select(.name == "preconditions") | .try[].script.content' \ + 'select(.name == "run") | .finally[] | select(.description == "system-bucket round-trip cleanup") | .script.content'; do + : >"$calls" + script=$(yq "$test | $selector" "$suite") + [ -n "$script" ] || { echo "no script at: $selector" >&2; false; } + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + grep -qE '^-n tenant-root delete backup sysbucket-backup( |$)' "$calls" || { + echo "the script at '$selector' does not delete Backup sysbucket-backup by name; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + done +} diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 03130cfb3..c48e32f24 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -439,7 +439,7 @@ spec: - name: preconditions try: - script: - timeout: 2m + timeout: 3m content: | set -eu if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then @@ -451,9 +451,11 @@ spec: exit 1 fi # Fixed names in a persistent namespace: a leftover Succeeded job - # would satisfy the waits in the run step without a new attempt. + # would satisfy the waits in the run step without a new attempt, and + # the driver adopts a leftover Backup of the BackupJob's name. kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s check: ($error == null): true @@ -562,10 +564,9 @@ spec: timeout: 3m content: | set -eu - backup=$(kubectl -n "$NAMESPACE" get backupjob sysbucket-backup --ignore-not-found -o jsonpath='{.status.backupRef.name}') kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s - if [ -n "$backup" ]; then - kubectl -n "$NAMESPACE" delete backup "$backup" --ignore-not-found --wait=true --timeout=60s - fi + # The driver names the Backup after the BackupJob; by name it is + # found even when status.backupRef was never written. + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s From dee5e6d8bf935874ad7648a9e8f63ff19809d21a Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:08:59 +0300 Subject: [PATCH 14/30] test(clickhouse): mark the round-trip's reachability poll as a fixed wait The probe loop sleeps between attempts because no Kubernetes condition reports that the host CoreDNS serves the rewrite and the Ingress serves the Test certificate. The e2e conventions allow such a sleep only under the marker that records it is waiting for something to model it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh index 88a3cf2b5..dc94a9c85 100755 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -103,6 +103,9 @@ while true; do kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true exit 1 fi + # TODO(e2e-replace-fixed-timeouts): no condition reports that the host + # CoreDNS answers with the rewrite and the Ingress serves the Test + # certificate, as seen from this namespace; poll until one does. sleep 10 done From 603114cd7c9954006e6b6cc92ea613cc7af1aab8 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:09:15 +0300 Subject: [PATCH 15/30] test(clickhouse): check the round-trip restored this run's sentinel row Every run inserted the same sentinel row and accepted any row after the restore, while the Altinity restore takes the newest remote archive of the release. A restore of an older run's archive therefore passed. The row now carries a per-run value and exactly that row has to come back. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../clickhouse/system-bucket-roundtrip.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh index dc94a9c85..73d2f0e63 100755 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -120,9 +120,12 @@ sidecar_before=$(sidecar_identity) [[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \ || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; } -log_step "Writing the sentinel row" +# Restore picks the newest remote backup of the release, so only a row no +# earlier run wrote shows that the archive restored is this run's. +sentinel="before-backup-$(date -u +%Y%m%dT%H%M%SZ)-${RANDOM}" +log_step "Writing the sentinel row ${sentinel}" clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" -clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')" +clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, '${sentinel}')" log_step "Backing up through cozy-default" kubectl apply -f - <= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; } +count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel WHERE name = '${sentinel}'" | tr -d '[:space:]') +[ "$count" = 1 ] || { log_error "after restore default.sentinel holds ${count} row(s) named ${sentinel}; expected 1"; exit 1; } sidecar_after=$(sidecar_identity) [ "$sidecar_after" = "$sidecar_before" ] \ || { log_error "the ClickHouse Pod or its clickhouse-backup container restarted during the round-trip (uid restarts: '${sidecar_before}' then '${sidecar_after}')"; exit 1; } -log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored" +log_success "System-bucket round-trip verified: sentinel row ${sentinel} restored" From 33608326f1046152cb587cf35924de82b7975044 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 5 Nov 2026 21:18:50 +0300 Subject: [PATCH 16/30] test(clickhouse): pin that an unreadable hold state keeps the job held The credentials hold returns the error when it cannot read the strategy Job or the projected Secret, and both the backup and the restore path pass it on, so the reconcile retries instead of starting the Job. Nothing tested that: swallowing either error, in the hold or at either call site, left the suite green while letting the Job start before the credentials reached the sidecar, which is the failure the hold exists to prevent. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../altinitystrategy_controller_test.go | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index e521d2d46..50e4cf865 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -9,6 +9,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -1041,3 +1042,135 @@ func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) { t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready) } } + +// A read the hold cannot complete must not release the Job: starting it +// before the credentials reach the sidecar is the failure the hold prevents. +func TestReconcileAltinity_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) { + cases := []struct { + name string + fail func(key client.ObjectKey, obj client.Object) bool + }{ + {name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isSecret := obj.(*corev1.Secret) + return isSecret && key.Name == "cozy-backups-creds" + }}, + {name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isJob := obj.(*batchv1.Job) + return isJob + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"), + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if tc.fail(key, obj) { + return apierrors.NewInternalError(errors.New("etcd unavailable")) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err == nil { + t.Errorf("reconcileAltinity returned no error although the %s could not be read", tc.name) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a strategy Job was created although the %s could not be read", tc.name) + } + }) + } +} + +func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) { + cases := []struct { + name string + fail func(key client.ObjectKey, obj client.Object) bool + }{ + {name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isSecret := obj.(*corev1.Secret) + return isSecret && key.Name == "cozy-backups-creds" + }}, + {name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isJob := obj.(*batchv1.Job) + return isJob + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if tc.fail(key, obj) { + return apierrors.NewInternalError(errors.New("etcd unavailable")) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err == nil { + t.Errorf("reconcileAltinityRestore returned no error although the %s could not be read", tc.name) + } + jobs := &batchv1.JobList{} + if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a restore Job was created although the %s could not be read", tc.name) + } + }) + } +} From 9376ba0078714440416a4a0a19d2955a61bcee30 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 17:56:59 +0300 Subject: [PATCH 17/30] fix(backups): hand every driver the same forcePathStyle backupStorage.forcePathStyle reaches drivers that read it differently. The controller, Velero and the ClickHouse sidecar parse it with strconv.ParseBool, the etcd and MongoDB strategies write it into a boolean field that Helm decodes as YAML 1.1, and the Kafka and RabbitMQ scripts take anything but the string "false" as path style. So "False" meant path style to Kafka and RabbitMQ only; "yes" meant path style to etcd, MongoDB, Kafka and RabbitMQ and was refused by Velero and the ClickHouse sidecar; "T" and "1" could not be applied to the boolean fields. The chart now accepts the spellings of strconv.ParseBool and of YAML 1.1 booleans, a superset of the boolean spellings the etcd and MongoDB strategies accepted, and passes every consumer "true" or "false", so they agree. Unset stays unset, so each driver keeps its own default. Any other value fails the render with a message naming it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../templates/_helpers.tpl | 25 + .../templates/deployment.yaml | 2 +- .../templates/strategy-etcd-default.yaml | 2 +- .../templates/strategy-kafka-default.yaml | 2 +- .../templates/strategy-mongodb-default.yaml | 2 +- .../templates/strategy-rabbitmq-default.yaml | 2 +- .../templates/velero-bsl.yaml | 2 +- .../tests/deployment_test.yaml | 9 + .../tests/force_path_style_test.yaml | 501 ++++++++++++++++++ 9 files changed, 541 insertions(+), 6 deletions(-) create mode 100644 packages/system/backupstrategy-controller/tests/force_path_style_test.yaml diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl index 6665cc639..106d65b35 100644 --- a/packages/system/backupstrategy-controller/templates/_helpers.tpl +++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl @@ -187,3 +187,28 @@ true {{- end -}} {{- end -}} {{- end -}} + +{{/* + backupStorage.forcePathStyle as "true", "false", or "" when unset. Its + consumers read it differently: the controller and Velero parse it with + strconv.ParseBool, the etcd and MongoDB strategies write it into a boolean + field that Helm decodes as YAML 1.1 (so "yes" or "off" is a boolean there), + and the Kafka and RabbitMQ scripts compare the string with "false". + Normalising it here means a spelling cannot mean path style to some drivers + and not to others. The spellings accepted are those of strconv.ParseBool and + of YAML 1.1 booleans, a superset of the boolean spellings the etcd and + MongoDB strategies accepted; any other value fails the render. +*/}} +{{- define "backupstrategy-controller.forcePathStyle" -}} +{{- $value := .Values.backupStorage.forcePathStyle -}} +{{- if not (kindIs "invalid" $value) -}} +{{- $raw := toString $value -}} +{{- if has $raw (list "1" "t" "T" "true" "True" "TRUE" "y" "Y" "yes" "Yes" "YES" "on" "On" "ON") -}} +true +{{- else if has $raw (list "0" "f" "F" "false" "False" "FALSE" "n" "N" "no" "No" "NO" "off" "Off" "OFF") -}} +false +{{- else if ne $raw "" -}} +{{- fail (printf "backupStorage.forcePathStyle must be a boolean or unset, got %q" $raw) -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index f35e97661..7d5c32fc4 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -47,7 +47,7 @@ spec: - name: BACKUP_STORAGE_REGION value: {{ .Values.backupStorage.region | quote }} - name: BACKUP_STORAGE_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} - name: BACKUP_STORAGE_SYSTEM_NAMESPACES value: {{ .Values.backupStorage.systemNamespaces | join "," | quote }} {{- if .Values.backupStorage.reconcileDefaultObjects }} diff --git a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml index f335f04cf..8d4b069a3 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml @@ -13,7 +13,7 @@ spec: endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} key: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}/" | quote }} region: {{ .Values.backupStorage.region | quote }} - forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} + forcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . }} # The etcd-operator reads AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY # from this Secret by default (no explicit key fields like the # MariaDB strategy's *SecretKeyRef). The projector writes exactly diff --git a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml index c65462295..93607fd32 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml @@ -67,7 +67,7 @@ spec: - name: S3_REGION value: {{ .Values.backupStorage.region | quote }} - name: S3_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} command: ["/usr/bin/bash", "-c"] # The driver script is kept as a standalone shell file and # embedded verbatim so hack/kafka-backup-script.bats can execute diff --git a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml index 7509f4bf7..12cc1d160 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml @@ -74,6 +74,6 @@ spec: prefix: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}" | quote }} region: {{ .Values.backupStorage.region | quote }} credentialsSecret: cozy-backups-creds - forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} + forcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . }} insecureSkipTLSVerify: {{ .Values.backupStorage.insecureSkipTLSVerify }} {{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml index 9c303a6d3..5f1e85db3 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml @@ -96,7 +96,7 @@ spec: - name: S3_REGION value: {{ .Values.backupStorage.region | quote }} - name: S3_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} command: ["/bin/sh", "-c"] args: - | diff --git a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml index e0ba05ad8..0621ebf0a 100644 --- a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml +++ b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml @@ -29,7 +29,7 @@ spec: key: cloud config: region: {{ .Values.backupStorage.region | quote }} - s3ForcePathStyle: {{ .Values.backupStorage.forcePathStyle | quote }} + s3ForcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} s3Url: {{ include "backupstrategy-controller.endpoint" . | quote }} {{- end }} {{- end }} diff --git a/packages/system/backupstrategy-controller/tests/deployment_test.yaml b/packages/system/backupstrategy-controller/tests/deployment_test.yaml index 6c17f5f58..f92455a8d 100644 --- a/packages/system/backupstrategy-controller/tests/deployment_test.yaml +++ b/packages/system/backupstrategy-controller/tests/deployment_test.yaml @@ -54,3 +54,12 @@ tests: name: BACKUP_STORAGE_SECRET_NAMESPACE value: tenant-root any: true + + # A value no consumer took as a boolean is refused once, at render time; see + # tests/force_path_style_test.yaml for what each consumer receives. + - it: "forcePathStyle that is no boolean spelling fails the render and names the value" + set: + backupStorage.forcePathStyle: "maybe" + asserts: + - failedTemplate: + errorPattern: 'backupStorage\.forcePathStyle must be a boolean or unset, got "maybe"' diff --git a/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml b/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml new file mode 100644 index 000000000..eda742951 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml @@ -0,0 +1,501 @@ +suite: forcePathStyle — one normalised boolean for every consumer + +# backupStorage.forcePathStyle reaches drivers that read it differently: the +# controller and Velero parse it with strconv.ParseBool, the etcd and MongoDB +# strategies write it into a boolean field Helm decodes as YAML 1.1, and the +# Kafka and RabbitMQ scripts compare the string with "false". The chart accepts +# the spellings of both strconv.ParseBool and YAML 1.1, hands each consumer +# "true" or "false", and refuses anything else, so a spelling cannot mean path +# style to one driver and not to another. + +templates: + - templates/deployment.yaml + - templates/strategy-etcd-default.yaml + - templates/strategy-mongodb-default.yaml + - templates/strategy-kafka-default.yaml + - templates/strategy-rabbitmq-default.yaml + - templates/velero-bsl.yaml + +release: + name: t + namespace: cozy-backup-controller + +set: + backupStorage.bucketNameOverride: test-bucket + +tests: + - it: "forcePathStyle '1' reaches the controller as true" + set: + backupStorage.forcePathStyle: "1" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 't' reaches the controller as true" + set: + backupStorage.forcePathStyle: "t" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'T' reaches every consumer as true" + set: + backupStorage.forcePathStyle: "T" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: true + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "true" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'true' reaches the controller as true" + set: + backupStorage.forcePathStyle: "true" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'True' reaches the controller as true" + set: + backupStorage.forcePathStyle: "True" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'TRUE' reaches the controller as true" + set: + backupStorage.forcePathStyle: "TRUE" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'y' reaches the controller as true" + set: + backupStorage.forcePathStyle: "y" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'Y' reaches the controller as true" + set: + backupStorage.forcePathStyle: "Y" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'yes' reaches every consumer as true" + set: + backupStorage.forcePathStyle: "yes" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: true + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "true" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'Yes' reaches the controller as true" + set: + backupStorage.forcePathStyle: "Yes" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'YES' reaches the controller as true" + set: + backupStorage.forcePathStyle: "YES" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'on' reaches the controller as true" + set: + backupStorage.forcePathStyle: "on" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'On' reaches the controller as true" + set: + backupStorage.forcePathStyle: "On" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'ON' reaches the controller as true" + set: + backupStorage.forcePathStyle: "ON" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle '0' reaches the controller as false" + set: + backupStorage.forcePathStyle: "0" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'f' reaches the controller as false" + set: + backupStorage.forcePathStyle: "f" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'F' reaches the controller as false" + set: + backupStorage.forcePathStyle: "F" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'false' reaches the controller as false" + set: + backupStorage.forcePathStyle: "false" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'False' reaches every consumer as false" + set: + backupStorage.forcePathStyle: "False" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: false + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "false" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'FALSE' reaches the controller as false" + set: + backupStorage.forcePathStyle: "FALSE" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'n' reaches the controller as false" + set: + backupStorage.forcePathStyle: "n" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'N' reaches the controller as false" + set: + backupStorage.forcePathStyle: "N" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'no' reaches the controller as false" + set: + backupStorage.forcePathStyle: "no" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'No' reaches the controller as false" + set: + backupStorage.forcePathStyle: "No" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'NO' reaches the controller as false" + set: + backupStorage.forcePathStyle: "NO" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'off' reaches every consumer as false" + set: + backupStorage.forcePathStyle: "off" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: false + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "false" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'Off' reaches the controller as false" + set: + backupStorage.forcePathStyle: "Off" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'OFF' reaches the controller as false" + set: + backupStorage.forcePathStyle: "OFF" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle as a YAML boolean (the default) reaches every consumer as true" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + + - it: "forcePathStyle as a YAML boolean false reaches the scripts as false" + set: + backupStorage.forcePathStyle: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + + - it: "forcePathStyle as the number 1 reaches the boolean fields as true" + set: + backupStorage.forcePathStyle: 1 + asserts: + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + + # Unset leaves each driver its own default, as before. + - it: "forcePathStyle null stays unset for every consumer" + set: + backupStorage.forcePathStyle: null + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: null + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: null + template: templates/strategy-mongodb-default.yaml + + - it: "forcePathStyle empty stays unset for every consumer" + set: + backupStorage.forcePathStyle: "" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: null + template: templates/strategy-etcd-default.yaml From 477540b8f163d791d245a8c0a0e48cdb36b33623 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:02:11 +0300 Subject: [PATCH 18/30] fix(backups): keep projecting credentials when forcePathStyle cannot be parsed The credentials projection carries every backup driver's keys, but only the ClickHouse sidecar's file needs forcePathStyle parsed as a boolean. A value that failed the parse failed the whole projection as a malformed source Secret, a terminal reason pointing at the wrong object, so CNPG, MariaDB, etcd and the other drivers lost their backups over a parse that serves one file. The value is now left out of that file and the reason is logged at error level with the namespace; the rest of the projection proceeds. The chart refuses such a value at render time, so this covers an environment set around it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/credentials_projector.go | 30 +++++++---- .../credentials_projector_test.go | 53 ++++++++++++++++--- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index ddf5ff8ef..576991d86 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" ) @@ -88,8 +89,8 @@ type BackupCredentialsConfig struct { TargetSecretName string Endpoint string Region string - // ForcePathStyle is the string "true"/"false" from - // backupStorage.forcePathStyle, projected into the credentials Secret + // ForcePathStyle is backupStorage.forcePathStyle as the chart + // normalises it ("true", "false", or "" when unset), projected into the credentials Secret // under the forcePathStyle key and into clickhouse-backup.yml. The app // charts cannot read backupStorage.* directly, so the ClickHouse // sidecar takes it from there. @@ -181,7 +182,14 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr } cloud := buildVeleroCredentialsFile(creds.accessKey, creds.secretKey) - clickHouseConfig, err := buildClickHouseBackupConfig(creds, cfg.ForcePathStyle) + clickHouseConfig, pathStyleErr, err := buildClickHouseBackupConfig(creds, cfg.ForcePathStyle) + if pathStyleErr != nil { + // The chart refuses such a value at render time, so only an env set + // around it gets here. It concerns the ClickHouse file alone, and + // failing the projection would take every other driver's + // credentials down with it. + log.FromContext(ctx).Error(pathStyleErr, "left force_path_style out of "+clickHouseBackupConfigKey, "namespace", targetNamespace) + } if err != nil { return &ProjectionError{ Reason: ReasonSourceMalformed, @@ -449,8 +457,9 @@ const clickHouseBackupConfigKey = "clickhouse-backup.yml" // buildClickHouseBackupConfig renders that file. The projected endpoint is // a bare host, and clickhouse-backup's S3 client rejects one without a -// scheme. -func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) ([]byte, error) { +// scheme. A forcePathStyle that is not a boolean is left out of the file +// and reported in pathStyleErr; err is a failure to render the file at all. +func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) (config []byte, pathStyleErr, err error) { s3 := map[string]any{ "bucket": creds.bucket, "endpoint": "https://" + creds.endpoint, @@ -463,13 +472,14 @@ func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) ([]by s3["region"] = creds.region } if forcePathStyle != "" { - pathStyle, err := strconv.ParseBool(forcePathStyle) - if err != nil { - return nil, fmt.Errorf("forcePathStyle %q is not a boolean", forcePathStyle) + if pathStyle, perr := strconv.ParseBool(forcePathStyle); perr != nil { + pathStyleErr = fmt.Errorf("forcePathStyle %q is not a boolean", forcePathStyle) + } else { + s3["force_path_style"] = pathStyle } - s3["force_path_style"] = pathStyle } - return yaml.Marshal(map[string]any{"s3": s3}) + config, err = yaml.Marshal(map[string]any{"s3": s3}) + return config, pathStyleErr, err } // stripScheme returns the endpoint stripped of any leading scheme prefix diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index 78ffb6c94..dd3cf131b 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -5,15 +5,19 @@ import ( "encoding/json" "errors" "fmt" + "strings" "testing" "time" + "github.com/go-logr/logr/funcr" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" ) @@ -746,14 +750,51 @@ func TestProject_ClickHouseBackupConfigOmitsAnEmptyRegion(t *testing.T) { } } -func TestProject_RejectsMalformedForcePathStyle(t *testing.T) { +// The parse only serves the ClickHouse file, while the same projection +// carries every other driver's credentials. An unparseable platform value +// must cost ClickHouse its path-style setting and say so, not fail the +// projection for all of them. +func TestProject_MalformedForcePathStyleSkipsOnlyTheClickHouseSetting(t *testing.T) { c := newFakeClient(flatSourceSecret()) + var logged []string + ctx := log.IntoContext(context.Background(), funcr.New(func(prefix, args string) { + logged = append(logged, args) + }, funcr.Options{})) cfg := defaultCfg() - cfg.ForcePathStyle = "yes please" - err := ProjectBackupCredentials(context.Background(), c, cfg, "tenant-acme") - var perr *ProjectionError - if !errors.As(err, &perr) || perr.Reason != ReasonSourceMalformed { - t.Fatalf("err = %v, want a %s ProjectionError", err, ReasonSourceMalformed) + cfg.ForcePathStyle = "yes" + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("projection failed for every driver over a ClickHouse-only setting: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + if string(got.Data["AWS_ACCESS_KEY_ID"]) != "AK" { + t.Errorf("AWS_ACCESS_KEY_ID = %q, want the projected key", got.Data["AWS_ACCESS_KEY_ID"]) + } + if string(got.Data["forcePathStyle"]) != "yes" { + t.Errorf("forcePathStyle = %q, want the platform value carried verbatim", got.Data["forcePathStyle"]) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err) + } + if doc.S3["access_key"] != "AK" { + t.Errorf("s3.access_key = %v, want the rest of the file written", doc.S3["access_key"]) + } + if pathStyle, set := doc.S3["force_path_style"]; set { + t.Errorf("s3.force_path_style = %v, want it left out", pathStyle) + } + found := false + for _, line := range logged { + if strings.Contains(line, `"error"`) && strings.Contains(line, `\"yes\"`) && strings.Contains(line, "force_path_style") { + found = true + } + } + if !found { + t.Errorf("no error naming the rejected value and the skipped setting was logged; got %q", logged) } } From c76080cc073d4282c4c968cc191c397e95be8e0a Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 17:04:28 +0300 Subject: [PATCH 19/30] test(clickhouse): pin the archive name across a credentials wait A backup's archive is named from the BackupJob's persisted StartedAt, and the restore of a Backup that records no archive picks one stamped no later than its takenAt. The credentials wait holds the Job after StartedAt is set and writes the job's status while it holds. Should a held reconcile restamp StartedAt, the Pod would write an archive the earlier reconciles never named; nothing caught that. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../altinitystrategy_controller_test.go | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 942957b7d..48592ac54 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -1495,3 +1495,79 @@ func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testi }) } } + +// A backup held for the credentials keeps the archive name it was given from +// its persisted StartedAt: the held reconciles write the job's status, and the +// Pod created after the hold, and the Backup recorded after it, must still +// name the same archive, which is stamped no later than the Backup's takenAt. +func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + secret := projectedCredentials("tenant-test", time.Now()) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityArchiveStrategy("clickhouse-strategy"), secret)) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + const want = "clickhouse-ch-test-20260304T050607Z" + reconcile := func(step string) { + t.Helper() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("%s: get BackupJob: %v", step, err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("%s: reconcileAltinity: %v", step, err) + } + } + + reconcile("held") + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); !apierrors.IsNotFound(err) { + t.Fatalf("the Job exists while the credentials wait should hold it: %v", err) + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + t.Fatalf("get projected Secret: %v", err) + } + secret.Annotations[credentialsUpdatedAtAnnotation] = time.Now().Add(-altinityCredentialsPropagation - time.Second).UTC().Format(time.RFC3339) + if err := r.Update(ctx, secret); err != nil { + t.Fatalf("age the credentials stamp: %v", err) + } + reconcile("after the wait") + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want { + t.Fatalf("backup Pod was rendered with %q after the wait, want --archive=%s", got, want) + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + reconcile("completed") + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want { + t.Errorf("Backup records archive %q, want %q", got, want) + } + if stamp := created.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout); stamp < "20260304T050607Z" { + t.Errorf("Backup takenAt %s is earlier than its archive's stamp", stamp) + } +} From 6280d06d010e8c1c829cad13aec74e8630f1cf48 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 17:47:51 +0300 Subject: [PATCH 20/30] test(clickhouse): describe the round-trip's guards by the archive-recording restore Restores now take the archive the Backup recorded rather than the newest one, and a leftover Backup of the BackupJob's name that records another archive fails the new BackupJob instead of being adopted. The comments justifying the sentinel row and the Backup cleanup, in the round-trip and in its unit test, still gave the earlier reasons, which would send the next reader after behaviour the driver no longer has. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/clickhouse-system-bucket-shared-state.bats | 6 +++--- hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml | 3 ++- hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index e76cc0a1d..ec4900cca 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -92,9 +92,9 @@ restore() { jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null } -# The Altinity driver names the Backup after the BackupJob and adopts one that -# already exists under that name, so a Backup left behind would stand in for -# the next run's backup. It is reclaimed by name, whether or not the BackupJob +# The Altinity driver names the Backup after the BackupJob, and a Backup left +# behind under that name records an earlier run's archive, which fails the next +# run's BackupJob. It is reclaimed by name, whether or not the BackupJob # recorded it in status.backupRef. @test "the round-trip reclaims its Backup by name before and after the run" { suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index c48e32f24..a4c4aef40 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -452,7 +452,8 @@ spec: fi # Fixed names in a persistent namespace: a leftover Succeeded job # would satisfy the waits in the run step without a new attempt, and - # the driver adopts a leftover Backup of the BackupJob's name. + # a leftover Backup of the BackupJob's name, recording an earlier + # run's archive, fails the new BackupJob. kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh index 73d2f0e63..c1c216389 100755 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -120,8 +120,8 @@ sidecar_before=$(sidecar_identity) [[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \ || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; } -# Restore picks the newest remote backup of the release, so only a row no -# earlier run wrote shows that the archive restored is this run's. +# Restore takes the archive this run's Backup recorded; a row no earlier run +# wrote is what shows the data restored is that archive's. sentinel="before-backup-$(date -u +%Y%m%dT%H%M%SZ)-${RANDOM}" log_step "Writing the sentinel row ${sentinel}" clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" From 1003cdb1b67d7a8ce2ee8ec6f6c627dadd638185 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:22:35 +0300 Subject: [PATCH 21/30] test(clickhouse): find the round-trip's Ingress Secret by its owner label alone kubectl refuses a resource name together with a label selector, and the restore asked for both with its stderr discarded, so the lookup always came back empty and the Secret the Test serves under the platform's S3 Ingress name was never deleted. The next run then stops on its own precondition that the Secret must not exist. A lookup that does fail now fails the restore, since an empty answer from a failed read is what kept this hidden. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- ...clickhouse-system-bucket-shared-state.bats | 58 ++++++++++++++++++- .../clickhouse/system-bucket-shared-state.sh | 8 ++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index ec4900cca..876e90954 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -18,8 +18,12 @@ setup() { metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"}, data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \ >"$STUB/coredns.json" + # The secret lookup follows kubectl's contract: a name together with a + # selector is refused, and a selector-only lookup prints the names in + # OWNED_SECRETS, the Secrets carrying the owner label. cat >"$STUB/kubectl" <<'EOF' #!/bin/bash +echo "$*" >>"$STUB/calls" case "$*" in *"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;; replace*) @@ -29,7 +33,29 @@ case "$*" in exit 1 fi ;; - *"get secret"*) ;; + *"get secret"*) + # Positionals are "get", "secret" and then any names. + words=0 selector= + while [ $# -gt 0 ]; do + case "$1" in + -l|--selector) selector=$2; shift 2 ;; + -n|--namespace|-o|--output) shift 2 ;; + -*) shift ;; + *) words=$((words + 1)); shift ;; + esac + done + if [ -n "${LOOKUP_FAILS:-}" ]; then + echo 'Unable to connect to the server: dial tcp: i/o timeout' >&2 + exit 1 + fi + if [ -n "$selector" ] && [ "$words" -gt 2 ]; then + echo 'error: name cannot be provided when a selector is specified' >&2 + exit 1 + fi + [ "$selector" = cozystack.io/e2e-owner=clickhouse-system-bucket ] || exit 0 + for s in ${OWNED_SECRETS:-}; do echo "secret/$s"; done + ;; + *"delete secret "*) ;; *) echo "unexpected kubectl $*" >&2; exit 2 ;; esac EOF @@ -74,6 +100,36 @@ restore() { [ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ] } +@test "the restore deletes the Ingress TLS Secret the Test created" { + OWNED_SECRETS=seaweedfs-s3-ingress-tls restore + grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || { + echo "restore_shared_state did not delete the Secret carrying the owner label; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + } +} + +@test "the restore leaves an Ingress TLS Secret without the owner label alone" { + for owned in '' e2e-other seaweedfs-s3-ingress-tls-old; do + : >"$STUB/calls" + OWNED_SECRETS=$owned restore + if grep -q 'delete secret' "$STUB/calls"; then + echo "restore_shared_state deleted a Secret although the labelled ones are '$owned'; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + fi + done +} + +@test "a Secret lookup that fails fails the restore" { + err=$(mktemp) + if LOOKUP_FAILS=1 restore 2>"$err"; then + echo "restore_shared_state returned 0 although it could not tell whether the Secret is left over" >&2 + cat "$err" >&2 + false + fi +} + @test "the run step's finally restores the shared objects" { # When the step ceiling fires, Chainsaw kills the shell and the EXIT trap # of system-bucket-roundtrip.sh never runs; the step's finally still does. diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh index 8275e8c13..acb577d27 100644 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh @@ -47,7 +47,13 @@ remove_dns_block() { restore_shared_state() { local rc=0 remove_dns_block || rc=1 - if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then + # kubectl refuses a name together with a selector, so the label is + # looked up on its own and the name matched in what comes back. + local owned + if ! owned=$(kubectl -n "$NAMESPACE" get secret -l "$OWNER_LABEL" -o name); then + log_error "could not list the Secrets labelled ${OWNER_LABEL} in ${NAMESPACE}; delete ${INGRESS_SECRET} by hand if it carries that label" + rc=1 + elif printf '%s\n' "$owned" | grep -qxF "secret/$INGRESS_SECRET"; then kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1 fi return "$rc" From 6e23bd5629da28edf63f60666fe2243c285b4ec1 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:22:43 +0300 Subject: [PATCH 22/30] test(clickhouse): delete the round-trip's Certificate before its Secret cert-manager issues the Secret of a Certificate that still exists again as soon as it is deleted, and the Certificate is only removed by Chainsaw's own cleanup after the step's finally has run. Deleting the Secret first therefore left a freshly issued copy behind in tenant-root, carrying no owner reference to take it down with the Certificate. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- ...clickhouse-system-bucket-shared-state.bats | 23 +++++++++++++++++++ .../clickhouse/chainsaw-test.yaml | 7 ++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index 876e90954..d03434531 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -179,3 +179,26 @@ STUBEOF } done } + +# cert-manager issues the Secret of a Certificate that still exists again as +# soon as it is deleted, so the Certificate has to go first or its Secret +# outlives the run. +@test "the round-trip cleanup deletes the Certificate before its Secret" { + script=$(yq 'select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[] + | select(.name == "run") | .finally[] + | select(.description == "system-bucket round-trip cleanup") | .script.content' \ + hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml) + [ -n "$script" ] || { echo "the run step has no system-bucket round-trip cleanup" >&2; false; } + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +STUBEOF + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + cert=$(grep -nE '^-n tenant-root delete certificate e2e-system-bucket-s3( |$)' "$STUB/calls" | cut -d: -f1) + secret=$(grep -nE '^-n tenant-root delete secret e2e-system-bucket-s3-tls( |$)' "$STUB/calls" | cut -d: -f1) + [ -n "$cert" ] && [ -n "$secret" ] && [ "$cert" -lt "$secret" ] || { + echo "the cleanup does not delete Certificate e2e-system-bucket-s3 before Secret e2e-system-bucket-s3-tls; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + } +} diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index a4c4aef40..774941ced 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -559,10 +559,12 @@ spec: ' # Chainsaw did not apply these: the jobs come from the script and the # Backup from the controller, and deleting a BackupJob reclaims no Backup. - # The Certificate's Secret outlives the Certificate Chainsaw deletes. + # The Certificate's Secret outlives the Certificate, and cert-manager + # issues it again while the Certificate exists, so the Certificate goes + # first rather than at Chainsaw's cleanup after this finally. - description: system-bucket round-trip cleanup script: - timeout: 3m + timeout: 4m content: | set -eu kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s @@ -570,4 +572,5 @@ spec: # The driver names the Backup after the BackupJob; by name it is # found even when status.backupRef was never written. kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s + kubectl -n "$NAMESPACE" delete certificate e2e-system-bucket-s3 --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s From 28f36134505e75d64adc5b3636fda198630412e9 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:28:53 +0300 Subject: [PATCH 23/30] test(clickhouse): delete the round-trip's TLS Secret in its own step's cleanup Chainsaw applied the Certificate, so the scripted delete of it in the run step's finally was hand-rolled teardown of a resource Chainsaw already reclaims. Chainsaw deletes a step's applied resources, waiting for them to go, before it runs that step's cleanup operations, which puts the Secret's delete after the Certificate's without any ordering written into a script. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- ...clickhouse-system-bucket-shared-state.bats | 36 +++++++++---------- .../clickhouse/chainsaw-test.yaml | 17 +++++---- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index d03434531..574eba84f 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -181,24 +181,24 @@ STUBEOF } # cert-manager issues the Secret of a Certificate that still exists again as -# soon as it is deleted, so the Certificate has to go first or its Secret -# outlives the run. -@test "the round-trip cleanup deletes the Certificate before its Secret" { - script=$(yq 'select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[] - | select(.name == "run") | .finally[] - | select(.description == "system-bucket round-trip cleanup") | .script.content' \ - hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml) - [ -n "$script" ] || { echo "the run step has no system-bucket round-trip cleanup" >&2; false; } - cat >"$STUB/kubectl" <<'STUBEOF' -#!/bin/bash -echo "$*" >>"$STUB/calls" -STUBEOF - PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" - cert=$(grep -nE '^-n tenant-root delete certificate e2e-system-bucket-s3( |$)' "$STUB/calls" | cut -d: -f1) - secret=$(grep -nE '^-n tenant-root delete secret e2e-system-bucket-s3-tls( |$)' "$STUB/calls" | cut -d: -f1) - [ -n "$cert" ] && [ -n "$secret" ] && [ "$cert" -lt "$secret" ] || { - echo "the cleanup does not delete Certificate e2e-system-bucket-s3 before Secret e2e-system-bucket-s3-tls; kubectl calls:" >&2 - cat "$STUB/calls" >&2 +# soon as it is deleted. Chainsaw deletes the Certificate it applied, and waits +# for it, before it runs the cleanup ops of the same step, so the Secret is +# deleted there and nowhere earlier. +@test "the Certificate's Secret is deleted after Chainsaw deletes the Certificate" { + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + applies=$(yq "$test | select(.name == \"create-certificate-and-probe\") | .try[] | select(has(\"apply\")) | .apply.file" "$suite") + [ "$applies" = system-bucket-tls.yaml ] || { + echo "step create-certificate-and-probe no longer applies system-bucket-tls.yaml alone: $applies" >&2 false } + deletes=$(yq -o=json -I=0 "$test | select(.name == \"create-certificate-and-probe\") | .cleanup" "$suite") + [ "$deletes" = '[{"delete":{"ref":{"apiVersion":"v1","kind":"Secret","name":"e2e-system-bucket-s3-tls"}}}]' ] || { + echo "the cleanup of step create-certificate-and-probe is not the one delete of Secret e2e-system-bucket-s3-tls: $deletes" >&2 + false + } + if yq "$test | (.try[], .finally[]?) | .script.content // \"\"" "$suite" | grep -qE 'delete +(secret +e2e-system-bucket-s3-tls|certificate +e2e-system-bucket-s3)( |$)'; then + echo "a step script deletes the Certificate or its Secret while Chainsaw still holds the Certificate" >&2 + false + fi } diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 774941ced..b71728720 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -485,6 +485,16 @@ spec: status: (conditions[?type == 'Ready']): - status: "True" + # The Certificate's Secret carries no owner reference, so it outlives the + # Certificate, and cert-manager issues it again while the Certificate + # exists. Chainsaw runs a step's cleanup ops after this step's cleaner has + # deleted the Certificate and waited for it. + cleanup: + - delete: + ref: + apiVersion: v1 + kind: Secret + name: e2e-system-bucket-s3-tls - name: create-clickhouse try: @@ -559,12 +569,9 @@ spec: ' # Chainsaw did not apply these: the jobs come from the script and the # Backup from the controller, and deleting a BackupJob reclaims no Backup. - # The Certificate's Secret outlives the Certificate, and cert-manager - # issues it again while the Certificate exists, so the Certificate goes - # first rather than at Chainsaw's cleanup after this finally. - description: system-bucket round-trip cleanup script: - timeout: 4m + timeout: 3m content: | set -eu kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s @@ -572,5 +579,3 @@ spec: # The driver names the Backup after the BackupJob; by name it is # found even when status.backupRef was never written. kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s - kubectl -n "$NAMESPACE" delete certificate e2e-system-bucket-s3 --ignore-not-found --wait=true --timeout=30s - kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s From d20b26d2ed9cbaabcb610825f085caeef8b92887 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:34:28 +0300 Subject: [PATCH 24/30] test(clickhouse): pin that a refused Secret delete fails the round-trip restore The owned Ingress Secret's delete only became reachable once its lookup stopped asking for a name and a selector together, and nothing failed the stub's delete, so dropping the delete's status from the restore's result passed every test. The CoreDNS half of the restore already has this guarantee pinned. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/clickhouse-system-bucket-shared-state.bats | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index 574eba84f..d09fa0260 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -55,7 +55,12 @@ case "$*" in [ "$selector" = cozystack.io/e2e-owner=clickhouse-system-bucket ] || exit 0 for s in ${OWNED_SECRETS:-}; do echo "secret/$s"; done ;; - *"delete secret "*) ;; + *"delete secret "*) + if [ -n "${DELETE_FAILS:-}" ]; then + echo 'Error from server (Forbidden): secrets "seaweedfs-s3-ingress-tls" is forbidden' >&2 + exit 1 + fi + ;; *) echo "unexpected kubectl $*" >&2; exit 2 ;; esac EOF @@ -121,6 +126,15 @@ restore() { done } +@test "a Secret delete the API server refuses fails the restore" { + err=$(mktemp) + if OWNED_SECRETS=seaweedfs-s3-ingress-tls DELETE_FAILS=1 restore 2>"$err"; then + echo "restore_shared_state returned 0 although kubectl delete secret was refused" >&2 + cat "$err" >&2 + false + fi +} + @test "a Secret lookup that fails fails the restore" { err=$(mktemp) if LOOKUP_FAILS=1 restore 2>"$err"; then From 4345a346ccdce59245777d93c0451a96aa1fd33f Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:37:38 +0300 Subject: [PATCH 25/30] test(clickhouse): check why the round-trip restore failed, not only that it did The refused-delete and failed-lookup cases asserted a non-zero status alone, which any earlier failure in the restore also produces, so a stub drifting from the script would have kept them green for the wrong reason. They now also require the delete to have been attempted and the lookup failure to be named. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/clickhouse-system-bucket-shared-state.bats | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index d09fa0260..10a869af6 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -133,6 +133,11 @@ restore() { cat "$err" >&2 false fi + grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || { + echo "restore_shared_state failed before it reached the Secret delete; kubectl calls:" >&2 + cat "$STUB/calls" "$err" >&2 + false + } } @test "a Secret lookup that fails fails the restore" { @@ -142,6 +147,11 @@ restore() { cat "$err" >&2 false fi + grep -q 'could not list the Secrets labelled' "$err" || { + echo "restore_shared_state failed without naming the Secret lookup:" >&2 + cat "$err" >&2 + false + } } @test "the run step's finally restores the shared objects" { From 0dc5a5473194bc8a0c112d313c9c2dc9eb5e52a5 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 13:49:42 +0300 Subject: [PATCH 26/30] test(clickhouse): remove the round-trip's TLS objects a killed run left Chainsaw deletes an applied object only when its own apply created it; one left behind by a run killed past its cleanup is patched and kept. The Certificate would then outlive every later run and keep re-issuing its Secret after each step cleanup, and a probe Pod whose sleep had ended never turns Ready, failing the next run's assert. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- ...clickhouse-system-bucket-shared-state.bats | 37 ++++++++++++++++++- .../clickhouse/chainsaw-test.yaml | 6 +++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index 10a869af6..e912be42f 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -221,8 +221,43 @@ STUBEOF echo "the cleanup of step create-certificate-and-probe is not the one delete of Secret e2e-system-bucket-s3-tls: $deletes" >&2 false } - if yq "$test | (.try[], .finally[]?) | .script.content // \"\"" "$suite" | grep -qE 'delete +(secret +e2e-system-bucket-s3-tls|certificate +e2e-system-bucket-s3)( |$)'; then + # The preconditions run before the apply, when Chainsaw holds no Certificate. + if yq "$test | select(.name != \"preconditions\") | (.try[], .finally[]?) | .script.content // \"\"" "$suite" | grep -qE 'delete +(secret +e2e-system-bucket-s3-tls|certificate +e2e-system-bucket-s3)( |$)'; then echo "a step script deletes the Certificate or its Secret while Chainsaw still holds the Certificate" >&2 false fi } + +# Chainsaw registers an applied object for deletion only when its apply created +# it; an object left by a run killed past its cleanup is patched and kept. The +# Certificate would then outlive every later run, re-issuing its Secret after +# each step cleanup, and a probe Pod whose `sleep` had ended never turns Ready +# again, so the preconditions remove all three before the apply. +@test "the preconditions remove the TLS objects and probe Pod a killed run left" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + : >"$calls" + script=$(yq "$test | select(.name == \"preconditions\") | .try[].script.content" "$suite") + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + line_of() { grep -nE "^-n tenant-root delete $1 --ignore-not-found --wait=true( |$)" "$calls" | head -n1 | cut -d: -f1; } + cert=$(line_of 'certificate e2e-system-bucket-s3') + secret=$(line_of 'secret e2e-system-bucket-s3-tls') + pod=$(line_of 'pod e2e-system-bucket-probe') + [ -n "$cert" ] && [ -n "$secret" ] && [ -n "$pod" ] || { + echo "the preconditions do not delete, and wait for, the Certificate, its Secret and the probe Pod; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + [ "$cert" -lt "$secret" ] || { + echo "the preconditions delete Secret e2e-system-bucket-s3-tls before its Certificate, which cert-manager then issues again" >&2 + false + } +} diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index b71728720..70fe6693a 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -458,6 +458,12 @@ spec: kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s + # Chainsaw reclaims only what its apply created, so objects a killed + # run left would be patched and kept. The Certificate goes before its + # Secret, which cert-manager issues again while the Certificate exists. + kubectl -n "$NAMESPACE" delete certificate e2e-system-bucket-s3 --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete pod e2e-system-bucket-probe --ignore-not-found --wait=true --timeout=60s check: ($error == null): true From 72a94ee426017bafb48e4a19aaac187e4c351cf2 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 14:26:02 +0300 Subject: [PATCH 27/30] test(clickhouse): give the round-trip's preconditions room for their deletes The step's ceiling SIGKILLs the shell and reports only "signal: killed", while a kubectl --timeout that fires first names the object that would not go. The deletes it runs now wait up to 270s in all, past the 3m ceiling, so a stuck delete late in the list would be reported by the ceiling instead of by kubectl. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 70fe6693a..e67b2674a 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -438,8 +438,11 @@ spec: steps: - name: preconditions try: + # The deletes below wait up to 270s in all (30+30+60+30+30+30+60), so a + # stuck one is named by its kubectl --timeout before the 5m ceiling kills + # the shell with no object named. A new delete needs its own --timeout. - script: - timeout: 3m + timeout: 5m content: | set -eu if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then From c26d2773a51d0ca471bd461c6a343a12a42c14a1 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 14:34:09 +0300 Subject: [PATCH 28/30] test(clickhouse): name what really leaves the round-trip's Secret behind The projected Secret the preconditions delete is left by whichever BackupJob ran in tenant-root before this Test, from this suite or another: Chainsaw does not fix the order of the Tests in a suite, so no particular one can be named. The restore budget is also not spent on a credentials hold: by then the stamp is older than the backup's own two-minute wait. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index e67b2674a..3c41dbdd3 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -393,9 +393,11 @@ spec: # bucket's SeaweedFS and its Ingress controller sit. # # The precondition step deletes tenant-root's projected cozy-backups-creds, -# which the mongodb and rabbitmq round-trips leave there after their -# cozy-default BackupJobs. Nothing reads it between suites, and the projector -# writes it again on the next BackupJob, which is the moment under test. +# which any BackupJob that ran in tenant-root before this Test leaves there: +# the projector runs for each BackupJob whatever its strategy, and Chainsaw +# does not fix the order of the Tests in a suite, so the earlier one may come +# from this suite or another. It writes the Secret again on the next +# BackupJob, which is the moment under test. apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -547,7 +549,8 @@ spec: # the Ingress serving the Test certificate, plus one last 30s # attempt and its 10s pause # 480s BackupJob to Succeeded, including the driver's 2m hold - # 480s RestoreJob to Succeeded, same hold + # 480s RestoreJob to Succeeded; no hold by then, the stamp is + # already older than the backup's own 2m wait # 90s the EXIT trap: Corefile replace plus a 60s Secret delete # 1330s (22m10s) # 25m leaves 2m50s for the SQL and kubectl calls between those waits, so From 8ba0380043c0eaaa4c01dcee8ae205702ed3f754 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 14:42:49 +0300 Subject: [PATCH 29/30] docs(clickhouse): name the clickhouse-backup handlers that reload the config clickhouse-backup reloads its config file in each handler that reaches S3, among them the create_remote, restore_remote and list calls the strategy makes. The status and action-log endpoints the strategy polls read in-memory state and do not reload it, so the mounted file reaches the sidecar on the calls that need credentials rather than on every request. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/credentials_projector.go | 7 ++++--- .../backupcontroller/credentials_projector_test.go | 3 ++- packages/apps/clickhouse/templates/clickhouse.yaml | 12 ++++++------ packages/apps/clickhouse/tests/backup_test.yaml | 7 ++++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index 576991d86..aaba89b3c 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -450,9 +450,10 @@ const credentialsUpdatedAtAnnotation = "backups.cozystack.io/credentials-updated // clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup // config. The ClickHouse chart mounts it as the sidecar's config file // instead of passing the values as environment: clickhouse-backup reloads -// the file on every API call and kubelet rewrites a mounted Secret in -// place, so credentials projected after the pod started, or rotated since, -// reach a running sidecar. +// the file in each API handler that reaches S3, among them the +// create_remote, restore_remote and list calls the strategy makes, and +// kubelet rewrites a mounted Secret in place, so credentials projected +// after the pod started, or rotated since, reach a running sidecar. const clickHouseBackupConfigKey = "clickhouse-backup.yml" // buildClickHouseBackupConfig renders that file. The projected endpoint is diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index dd3cf131b..74fd936d1 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -673,7 +673,8 @@ func TestProject_HappyPath_HTTPEndpoint(t *testing.T) { // The ClickHouse sidecar reads its S3 settings from this key as a mounted // file, which kubelet refreshes in place: clickhouse-backup reloads its -// config file on every API call, while its environment is fixed at start. +// config file in each API handler that reaches S3, while its environment +// is fixed at start. func TestProject_WritesClickHouseBackupConfig(t *testing.T) { cases := []struct { name string diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 4934e8db1..5d0c1c6ab 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -230,12 +230,12 @@ spec: backupStorage.forcePathStyle included, comes from the clickhouse-backup.yml key the platform projects into cozy-backups-creds, mounted below as the config file. - clickhouse-backup reloads that file on every API call and - kubelet rewrites a mounted Secret in place, so credentials - projected after the Pod started, or rotated since, reach - the running sidecar; environment would be read only at - start and would override the file, so none of those is - set here. + clickhouse-backup reloads that file in each API handler + that reaches S3 and kubelet rewrites a mounted Secret in + place, so credentials projected after the Pod started, or + rotated since, reach the running sidecar; environment + would be read only at start and would override the file, + so none of those is set here. S3_FORCE_PATH_STYLE selects path-style vs vhost-style S3 addressing. On the legacy paths (own Secret via diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml index 7448e02a3..7954d5d3d 100644 --- a/packages/apps/clickhouse/tests/backup_test.yaml +++ b/packages/apps/clickhouse/tests/backup_test.yaml @@ -47,9 +47,10 @@ tests: - equal: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_PATH")].value value: tenant-test/ch-test - # clickhouse-backup reloads this file on every API call and kubelet - # rewrites the mounted Secret in place, so credentials projected after - # the Pod started, or rotated since, reach the running sidecar. + # clickhouse-backup reloads this file in each API handler that reaches + # S3 and kubelet rewrites the mounted Secret in place, so credentials + # projected after the Pod started, or rotated since, reach the running + # sidecar. - equal: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="CLICKHOUSE_BACKUP_CONFIG")].value value: /etc/clickhouse-backup-s3/config.yml From 35823c6c9c4d960f6417ad6666f8d2ca8b59b996 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 14:56:26 +0300 Subject: [PATCH 30/30] test(clickhouse): list the system-bucket round-trip in the suite header The header names the backup Tests of the suite and why they are split; it still described two of them after the third was added, so a reader starting from it would miss the only Test that covers the system-bucket path. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 3c41dbdd3..e6253c06a 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -91,7 +91,7 @@ spec: status: replicas: 1 --- -# ClickHouse Altinity backup/restore E2E — the two backup Tests below sit in +# ClickHouse Altinity backup/restore E2E — the three backup Tests below sit in # this suite alongside the plain `clickhouse` Test above. Chainsaw port of the # intended hack/e2e-apps/backup-clickhouse.bats (cozystack#2600), shaped like the # etcd/mariadb/postgres suites: @@ -107,8 +107,15 @@ spec: # driving the validated example scripts # (examples/backups/clickhouse). Also runs in # CI, in tenant-root. +# clickhouse-3-system-bucket-roundtrip +# — backup and in-place restore through the +# shipped cozy-default BackupClass and the +# platform system bucket, for a release +# whose Pod started before its credentials +# were projected. Runs in CI, in tenant-root. # -# Why the split: the two Tests answer different questions at different cost. The +# Why the split: the first two Tests answer different questions at different +# cost, and the third covers the system-bucket path neither of them takes. The # contracts Test needs no object store, so it stays in the default tenant-test # namespace next to the other app suites and is the seconds-long guard on the # chart wiring the Altinity strategy depends on — the sidecar container/port and