From e160a0dc8799fcd30abaf8523ff77c66a4e01895 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 13 Jul 2026 15:41:59 +0500 Subject: [PATCH] fix(backups): derive default S3 endpoint from the provisioned bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default backupStorage.endpoint is http://seaweedfs-s3...svc:8333, but Cozystack ships SeaweedFS with global.seaweedfs.enableSecurity=true, so its in-cluster S3 serves TLS on :8333 behind the self-signed "SeaweedFS CA". Every cozy-default backup (etcd/mariadb/velero/fdb/cnpg) therefore hits a TLS listener over plaintext and fails the handshake. This is fatal for the etcd v1alpha2 adoption migration: its mandatory pre-upgrade safety snapshot writes to this endpoint, so migration 50 hard-fails and blocks the 1.5->1.6 upgrade on any cluster with a legacy etcd. The Etcd Strategy S3 schema has no caCert/insecureSkipVerify field, so it cannot target the self-signed in-cluster endpoint at all. Add a "backupstrategy-controller.endpoint" helper that, for a provisioned bucket, resolves the endpoint from the COSI bucket's system credentials Secret (backupStorage.systemSecretName) — the external S3 ingress with an ACME cert, the same trusted endpoint COSI advertises and every backup operator can verify — and forces the https:// scheme. All Strategy CRs, the Velero BackupStorageLocation and the controller Deployment env are routed through it. It falls back to .Values.backupStorage.endpoint for external S3 (provisionBucket=false) and for offline `helm template`/unit renders and the pre-reconcile first install, where the Secret lookup returns nothing (Flux re-renders on spec.interval once the Secret exists). helm-unittest extended to assert every consumer picks up the resolved endpoint (CNPG/Etcd/Velero keep the full URL, MariaDB/FDB strip the scheme and derive their secure flag) plus the external-S3 verbatim fallback. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Myasnikov Daniil --- .../templates/_helpers.tpl | 41 ++++++++++++++ .../templates/deployment.yaml | 2 +- .../templates/strategy-cnpg-default.yaml | 2 +- .../templates/strategy-etcd-default.yaml | 2 +- .../strategy-foundationdb-default.yaml | 5 +- .../templates/strategy-mariadb-default.yaml | 5 +- .../templates/velero-bsl.yaml | 2 +- .../tests/endpoint_form_test.yaml | 55 +++++++++++++++++-- .../backupstrategy-controller/values.yaml | 25 +++++++-- 9 files changed, 120 insertions(+), 19 deletions(-) diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl index b0b275136..577b2e593 100644 --- a/packages/system/backupstrategy-controller/templates/_helpers.tpl +++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl @@ -60,3 +60,44 @@ BucketClaim CAN come into existence even on the first install. */}} {{- end -}} {{- end -}} + +{{/* + Resolve the S3 endpoint URL (scheme included) shared by every default + Strategy CR (CNPG/Etcd/MariaDB/FDB), the Velero BSL, and the controller + Deployment env. + + Why not just use .Values.backupStorage.endpoint: Cozystack ships SeaweedFS + with global.seaweedfs.enableSecurity=true, so its in-cluster S3 Service + serves TLS on :8333 fronted by the self-signed "SeaweedFS CA". The static + default endpoint (http://seaweedfs-s3...svc:8333) therefore hits a TLS + listener over plaintext and every backup upload fails the handshake. The + Etcd Strategy's S3 schema has no caCert/insecureSkipVerify field, so it + cannot target the self-signed in-cluster endpoint at all — it needs a + trusted-cert endpoint. The COSI-provisioned bucket exposes exactly that: the + external S3 ingress (ACME cert), advertised in the bucket's system + credentials Secret (backupStorage.systemSecretName) — the same Secret the + projector already consumes. We read the bucket host from there and force the + https:// scheme (the S3 ingress is always TLS). + + Failure semantics mirror bucketName: + - provisionBucket: false → external S3, the admin-configured + .Values.backupStorage.endpoint is authoritative. Return it as-is. + - provisionBucket: true + system Secret present → return + https://. + - provisionBucket: true + Secret missing (offline `helm template`/unit + render, or pre-reconcile install where lookup returns nil) → fall back + to .Values.backupStorage.endpoint. On a live deploy Flux re-renders on + spec.interval once the Secret exists, promoting the derived endpoint. +*/}} +{{- define "backupstrategy-controller.endpoint" -}} +{{- if not .Values.backupStorage.provisionBucket -}} +{{- .Values.backupStorage.endpoint -}} +{{- else -}} +{{- $secret := lookup "v1" "Secret" .Values.backupStorage.namespace .Values.backupStorage.systemSecretName -}} +{{- if and $secret $secret.data (index $secret.data "endpoint") -}} +{{- printf "https://%s" (b64dec (index $secret.data "endpoint") | trimPrefix "https://" | trimPrefix "http://") -}} +{{- else -}} +{{- .Values.backupStorage.endpoint -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index 42adb19a8..7c89f416f 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -43,7 +43,7 @@ spec: - name: BACKUP_STORAGE_CREDS_SECRET_NAME value: cozy-backups-creds - name: BACKUP_STORAGE_ENDPOINT - value: {{ .Values.backupStorage.endpoint | quote }} + value: {{ include "backupstrategy-controller.endpoint" . | quote }} - name: BACKUP_STORAGE_REGION value: {{ .Values.backupStorage.region | quote }} - name: BACKUP_STORAGE_FORCE_PATH_STYLE diff --git a/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml index 10aada2cf..35a8e6545 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml @@ -9,7 +9,7 @@ spec: serverName: {{ printf "{{ .Application.metadata.namespace }}-{{ .Application.metadata.name }}" | quote }} barmanObjectStore: destinationPath: {{ printf "s3://%s/{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}/" $bucketName | quote }} - endpointURL: {{ .Values.backupStorage.endpoint | quote }} + endpointURL: {{ include "backupstrategy-controller.endpoint" . | quote }} retentionPolicy: "30d" s3Credentials: secretRef: diff --git a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml index 70eb8f52b..30b740f22 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml @@ -9,7 +9,7 @@ spec: destination: s3: bucket: {{ $bucketName | quote }} - endpoint: {{ .Values.backupStorage.endpoint | quote }} + endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} key: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}/" | quote }} region: {{ .Values.backupStorage.region | quote }} forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} diff --git a/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml index 5d72101cf..306c62aba 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml @@ -1,5 +1,6 @@ -{{- $endpointHost := .Values.backupStorage.endpoint | trimPrefix "https://" | trimPrefix "http://" -}} -{{- $secureConn := ternary "1" "0" (hasPrefix "https://" .Values.backupStorage.endpoint) -}} +{{- $endpoint := include "backupstrategy-controller.endpoint" . -}} +{{- $endpointHost := $endpoint | trimPrefix "https://" | trimPrefix "http://" -}} +{{- $secureConn := ternary "1" "0" (hasPrefix "https://" $endpoint) -}} {{- $bucketName := include "backupstrategy-controller.bucketName" . -}} {{- if $bucketName -}} apiVersion: strategy.backups.cozystack.io/v1alpha1 diff --git a/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml index 6ed3661fd..9a222a895 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml @@ -1,4 +1,5 @@ -{{- $endpointHost := .Values.backupStorage.endpoint | trimPrefix "https://" | trimPrefix "http://" -}} +{{- $endpoint := include "backupstrategy-controller.endpoint" . -}} +{{- $endpointHost := $endpoint | trimPrefix "https://" | trimPrefix "http://" -}} {{- $bucketName := include "backupstrategy-controller.bucketName" . -}} {{- if $bucketName -}} apiVersion: strategy.backups.cozystack.io/v1alpha1 @@ -20,6 +21,6 @@ spec: name: cozy-backups-creds key: AWS_SECRET_ACCESS_KEY tls: - enabled: {{ hasPrefix "https://" .Values.backupStorage.endpoint }} + enabled: {{ hasPrefix "https://" $endpoint }} compression: gzip {{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml index a08153995..e0ba05ad8 100644 --- a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml +++ b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml @@ -30,6 +30,6 @@ spec: config: region: {{ .Values.backupStorage.region | quote }} s3ForcePathStyle: {{ .Values.backupStorage.forcePathStyle | quote }} - s3Url: {{ .Values.backupStorage.endpoint | quote }} + s3Url: {{ include "backupstrategy-controller.endpoint" . | quote }} {{- end }} {{- end }} diff --git a/packages/system/backupstrategy-controller/tests/endpoint_form_test.yaml b/packages/system/backupstrategy-controller/tests/endpoint_form_test.yaml index 12ba39535..0c569ec2c 100644 --- a/packages/system/backupstrategy-controller/tests/endpoint_form_test.yaml +++ b/packages/system/backupstrategy-controller/tests/endpoint_form_test.yaml @@ -1,13 +1,25 @@ suite: endpoint scheme → per-driver derivations (TLS / secure_connection / scheme stripping) -# backupStorage.endpoint is a single URL adapted per driver. MariaDB and -# FoundationDB want a bare host:port and derive their secure flag from the -# scheme, so flipping http<->https must flip tls.enabled / secure_connection -# without leaking the scheme into the host fields. +# The S3 endpoint is resolved by the "backupstrategy-controller.endpoint" +# helper and shared by every driver: CNPG/Etcd want a full URL (scheme +# included), MariaDB/FoundationDB want a bare host:port and derive their secure +# flag from the scheme, and the Velero BSL wants a full URL. Flipping +# http<->https must flip tls.enabled / secure_connection without leaking the +# scheme into the host fields, and must reach every consumer identically. +# +# NOTE: the provisionBucket:true derivation (read the external ACME endpoint +# from the COSI system Secret and force https://) uses `lookup`, which returns +# nothing under `helm template` / helm-unittest. These offline tests therefore +# exercise the FALLBACK path — where the helper returns +# .Values.backupStorage.endpoint verbatim — which is exactly what runs on the +# pre-reconcile first install and for external S3 (provisionBucket:false). templates: + - templates/strategy-etcd-default.yaml + - templates/strategy-cnpg-default.yaml - templates/strategy-mariadb-default.yaml - templates/strategy-foundationdb-default.yaml + - templates/velero-bsl.yaml release: name: t @@ -18,10 +30,22 @@ set: backupStorage.bucketNameOverride: test-bucket tests: - - it: "https endpoint: MariaDB TLS on, FDB secure_connection=1, scheme stripped from host fields" + - it: "https endpoint: full URL reaches CNPG/Etcd/Velero; MariaDB TLS on, FDB secure_connection=1, scheme stripped from host fields" set: backupStorage.endpoint: https://s3.example.com:8333 asserts: + - equal: + path: spec.template.destination.s3.endpoint + value: https://s3.example.com:8333 + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.barmanObjectStore.endpointURL + value: https://s3.example.com:8333 + template: templates/strategy-cnpg-default.yaml + - equal: + path: spec.config.s3Url + value: https://s3.example.com:8333 + template: templates/velero-bsl.yaml - equal: path: spec.template.storage.s3.endpoint value: s3.example.com:8333 @@ -39,10 +63,14 @@ tests: content: secure_connection=1 template: templates/strategy-foundationdb-default.yaml - - it: "http endpoint: MariaDB TLS off, FDB secure_connection=0" + - it: "http endpoint: MariaDB TLS off, FDB secure_connection=0, Etcd keeps scheme" set: backupStorage.endpoint: http://s3.example.com:8333 asserts: + - equal: + path: spec.template.destination.s3.endpoint + value: http://s3.example.com:8333 + template: templates/strategy-etcd-default.yaml - equal: path: spec.template.storage.s3.tls.enabled value: false @@ -51,3 +79,18 @@ tests: path: spec.template.blobStoreConfiguration.urlParameters content: secure_connection=0 template: templates/strategy-foundationdb-default.yaml + + - it: "external S3 (provisionBucket:false): endpoint is used verbatim (no derivation)" + set: + backupStorage.provisionBucket: false + backupStorage.bucketName: ext-bucket + backupStorage.endpoint: https://minio.example.internal:9000 + asserts: + - equal: + path: spec.template.destination.s3.endpoint + value: https://minio.example.internal:9000 + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.config.s3Url + value: https://minio.example.internal:9000 + template: templates/velero-bsl.yaml diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index d36635507..24c5c1bba 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -47,11 +47,26 @@ backupStorage: # name independent of .bucketName above, and the BucketClaim status # is the only authoritative source. bucketNameOverride: "" - # endpoint is wired into every default Strategy CR at chart-render time. - # Switching to an https:// URL silently enables TLS in the MariaDB - # strategy (tls.enabled is computed from the scheme). Make sure the CA - # bundle is reachable to the operator (mariadb's tls.caSecretKeyRef, the - # FDB pod ca-cert volume, Velero's BSL caCert field) before flipping it. + # endpoint is the S3 endpoint URL shared by every default Strategy CR, the + # Velero BSL and the controller Deployment. For a provisioned bucket + # (provisionBucket: true, the default) it is NOT used verbatim: the + # "backupstrategy-controller.endpoint" helper derives the real endpoint from + # the COSI bucket's system credentials Secret (backupStorage.systemSecretName) + # and forces https://, because Cozystack's SeaweedFS runs with + # enableSecurity=true and serves TLS on :8333 behind the self-signed + # "SeaweedFS CA". The COSI-advertised endpoint is the external S3 ingress + # (ACME cert), which every consumer can verify — unlike the in-cluster + # service, whose CA is not plumbed to the backup operators and which the Etcd + # Strategy's S3 schema (no caCert field) cannot target at all. + # + # This value is the FALLBACK, used only when the derivation cannot run: + # - provisionBucket: false (external S3) — set it to your S3 endpoint, + # scheme included; it is used verbatim (the MariaDB/FDB strategies compute + # their secure flag from the scheme, so https:// enables TLS there — make + # the CA bundle reachable to those operators before pointing at a + # self-signed external endpoint); + # - offline `helm template` / unit renders and the pre-reconcile first + # install, where the Secret lookup returns nothing. endpoint: http://seaweedfs-s3.tenant-root.svc.cozy.local:8333 # region is re-projected into cozy-backups-creds on the next reconcile # (1-minute tick for system namespaces, on-demand for tenant namespaces).