From aced011ab704122e0ba2e9f341b0b6d06d5ea747 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 22 Jun 2026 18:16:15 +0300 Subject: [PATCH] feat(platform): add opt-in in-cluster east-west encryption toggle Add networking.encryption.enabled (off by default), a platform-level opt-in that turns on in-cluster transport encryption for the active datapath. On the default kubeovn-cilium datapath (and its generic variant) KubeOVN owns the overlay, so the toggle maps to KubeOVN's Geneve-overlay IPsec (kube-ovn ENABLE_OVN_IPSEC). KubeOVN's controller provisions the IPsec CA and per-node certificates through the Kubernetes CSR API, so no cert-manager wiring is required. This is defense-in-depth for the whole mesh without per-application changes; it complements, and does not replace, application-level / end-to-end TLS. Values comments document the per-datapath scope (including Kilo's cross-location-only mesh and the unreached Cilium-native path on cilium/cilium-generic variants) and the MTU/throughput overhead. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../platform/templates/bundles/system.yaml | 15 ++ .../bundles_networking_encryption_test.yaml | 137 ++++++++++++++++++ packages/core/platform/values.yaml | 38 +++++ 3 files changed, 190 insertions(+) create mode 100644 packages/core/platform/tests/bundles_networking_encryption_test.yaml diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 84aff47ac..de05114e1 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -13,6 +13,15 @@ {{- if .Values.networking.kubeovn.MASTER_NODES -}} {{- $_ := set $kubeovnDict "MASTER_NODES" .Values.networking.kubeovn.MASTER_NODES -}} {{- end -}} +{{- /* Opt-in east-west encryption (networking.encryption.enabled): on the + default kubeovn-cilium datapath the overlay is KubeOVN's Geneve + tunnel, so encrypting it means turning on KubeOVN IPsec. KubeOVN's + own controller provisions the IPsec CA and per-node certs through + the Kubernetes CSR API — no cert-manager wiring needed here. Off by + default; complements (does not replace) application-level TLS. */ -}} +{{- if dig "encryption" "enabled" false .Values.networking -}} +{{- $_ := set $kubeovnDict "func" (dict "ENABLE_OVN_IPSEC" true) -}} +{{- end -}} {{- $kubeovnValues := dict "kube-ovn" $kubeovnDict -}} {{- $_ := set $networkingComponents "kubeovn" (dict "values" $kubeovnValues) -}} {{- /* For Talos (isp-full): use KubePrism endpoint and disable cgroup autoMount */ -}} @@ -79,6 +88,12 @@ {{- end -}} {{- /* For generic k8s (k3s), control-plane label has value "true" */ -}} {{- $_ := set $kubeovnDict "MASTER_NODES_LABEL" "node-role.kubernetes.io/control-plane=true" -}} +{{- /* Opt-in east-west encryption (networking.encryption.enabled): same + KubeOVN Geneve-overlay IPsec path as isp-full. See the isp-full + branch above for the mechanism notes. */ -}} +{{- if dig "encryption" "enabled" false .Values.networking -}} +{{- $_ := set $kubeovnDict "func" (dict "ENABLE_OVN_IPSEC" true) -}} +{{- end -}} {{- $kubeovnValues := dict "kube-ovn" $kubeovnDict -}} {{- $_ := set $networkingComponents "kubeovn" (dict "values" $kubeovnValues) -}} {{- /* Cilium configuration - for generic k8s, always enable cgroup autoMount */ -}} diff --git a/packages/core/platform/tests/bundles_networking_encryption_test.yaml b/packages/core/platform/tests/bundles_networking_encryption_test.yaml new file mode 100644 index 000000000..3ec3efaad --- /dev/null +++ b/packages/core/platform/tests/bundles_networking_encryption_test.yaml @@ -0,0 +1,137 @@ +suite: networking.encryption bundle wiring +templates: + - templates/bundles/system.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: leaves KubeOVN overlay IPsec unset on the default isp-full datapath when encryption is off + set: + bundles: + system: + enabled: true + variant: isp-full + networking: + encryption: + enabled: false + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.variant + value: kubeovn-cilium + - notExists: + path: spec.components.kubeovn.values.kube-ovn.func.ENABLE_OVN_IPSEC + + - it: leaves KubeOVN overlay IPsec unset on isp-full when networking.encryption is absent (backward compatible, no nil-pointer) + set: + bundles: + system: + enabled: true + variant: isp-full + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.variant + value: kubeovn-cilium + - notExists: + path: spec.components.kubeovn.values.kube-ovn.func.ENABLE_OVN_IPSEC + + - it: enables KubeOVN Geneve-overlay IPsec on the default isp-full datapath when encryption is on + set: + bundles: + system: + enabled: true + variant: isp-full + networking: + encryption: + enabled: true + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.variant + value: kubeovn-cilium + - equal: + path: spec.components.kubeovn.values.kube-ovn.func.ENABLE_OVN_IPSEC + value: true + + - it: enables KubeOVN overlay IPsec on the generic kubeovn-cilium-generic datapath when encryption is on + set: + bundles: + system: + enabled: true + variant: isp-full-generic + networking: + encryption: + enabled: true + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.variant + value: kubeovn-cilium-generic + - equal: + path: spec.components.kubeovn.values.kube-ovn.func.ENABLE_OVN_IPSEC + value: true + + - it: leaves the generic datapath IPsec unset when encryption is off + set: + bundles: + system: + enabled: true + variant: isp-full-generic + networking: + encryption: + enabled: false + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.variant + value: kubeovn-cilium-generic + - notExists: + path: spec.components.kubeovn.values.kube-ovn.func.ENABLE_OVN_IPSEC + + - it: keeps the existing kube-ovn ipv4 component overrides alongside the IPsec toggle (deep merge, not replace) + set: + bundles: + system: + enabled: true + variant: isp-full + networking: + encryption: + enabled: true + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.components.kubeovn.values.kube-ovn.ipv4.POD_CIDR + value: 10.244.0.0/16 + - equal: + path: spec.components.kubeovn.values.kube-ovn.func.ENABLE_OVN_IPSEC + value: true + + - it: renders without a nil-pointer error when networking is set to null and encryption is therefore unreachable + set: + bundles: + system: + enabled: true + variant: isp-full + networking: null + documentSelector: + path: metadata.name + value: cozystack.networking + asserts: + - equal: + path: spec.variant + value: kubeovn-cilium + - notExists: + path: spec.components diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index a5ce86d4c..99be6416e 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -45,6 +45,44 @@ networking: # Set this to comma-separated list of master node IPs to override. kubeovn: MASTER_NODES: "" + # In-cluster (east-west) transport encryption. Opt-in, OFF by default. + # + # When enabled, pod-to-pod traffic that crosses nodes is encrypted at the + # network layer by the ACTIVE datapath, regardless of whether the workload + # speaks TLS — defense-in-depth for the whole mesh without per-application + # changes. This COMPLEMENTS, and does NOT replace, application-level / + # end-to-end TLS: it never reaches an external client and gives a tenant no + # verifiable endpoint certificate (see the per-app certificate series). Use + # it alongside, not instead of, TLS on exposed endpoints. + # + # The mechanism is datapath-specific (selected by bundles.system.variant): + # + # * kubeovn-cilium (isp-full, the default) and kubeovn-cilium-generic + # (isp-full-generic): KubeOVN owns the datapath; its Geneve overlay is + # encrypted with IPsec (kube-ovn ENABLE_OVN_IPSEC). KubeOVN's own + # controller provisions the IPsec CA and per-node certificates through + # the Kubernetes CSR API — no cert-manager wiring is required. This is + # the path this toggle drives today. + # * cilium / cilium-generic (Cilium owns the datapath): Cilium-native + # WireGuard/IPsec is the right mechanism there, but the platform system + # bundle does not select those variants, so this toggle does not reach + # them yet. + # * cilium-kilo: Kilo already establishes a WireGuard mesh, but only at + # meshGranularity=location — it encrypts CROSS-LOCATION traffic and + # leaves intra-location pod-to-pod in cleartext by design. This toggle + # targets exactly that intra-location gap and composes with Kilo rather + # than conflicting with it. + # + # CAVEATS: encrypting the overlay adds per-packet ESP overhead and lowers + # the effective MTU, so east-west throughput drops and latency rises; the + # cost is highest for high-bandwidth flows. IPsec here protects the + # inter-node overlay tunnels — same-node pod-to-pod traffic never leaves the + # host and is not tunneled or encrypted. Flipping this on an existing + # cluster triggers a rollout of the CNI components. On variants without a + # cozystack-managed encryptable datapath (e.g. isp-hosted / noop) the toggle + # has no effect. + encryption: + enabled: false # Service publishing and ingress configuration publishing: host: "example.org"