From 078e9f0daa2aff667cec1f22676b6549d461c0ea Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 8 Oct 2026 18:15:35 +0200 Subject: [PATCH] fix(keycloak): require a persistent DEK set for the KMS proxy Without encryption.deksetSecretName the proxy mints a new DEK set on every start and keeps it only in memory. After any restart of the proxy pod it can no longer decrypt the rows it wrote before, so the default configuration loses access to all encrypted user data on the first rollout, drain or OOM. Refuse to render the proxy without a DEK set unless encryption.fields is "disabled". Passthrough keeps a way for the static backend to get its KEK Secret created before the DEK set is minted with it. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- packages/system/keycloak/templates/proxy.yaml | 11 ++++ .../keycloak/tests/encryption_test.yaml | 64 ++++++++++++++++++- .../system/keycloak/tests/placement_test.yaml | 3 + packages/system/keycloak/values.yaml | 24 ++++--- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/packages/system/keycloak/templates/proxy.yaml b/packages/system/keycloak/templates/proxy.yaml index 8cc96892e..d75a725c4 100644 --- a/packages/system/keycloak/templates/proxy.yaml +++ b/packages/system/keycloak/templates/proxy.yaml @@ -55,6 +55,17 @@ {{- if and (gt $replicas 1) (not .Values.encryption.deksetSecretName) }} {{- fail "encryption.replicas > 1 requires encryption.deksetSecretName (a shared wrapped DEK set); otherwise replicas would encrypt deterministic columns under different keys" }} {{- end }} +{{- /* + Without KKP_DEKSET_FILE the proxy mints a fresh DEK set on every start and + keeps it only in memory, so after any restart (rollout, drain, OOM) it can + no longer decrypt what it wrote before. Refuse to encrypt without a + persistent DEK set. fields=disabled is pure passthrough and needs no key: it + is how the static backend gets its KEK Secret created before the operator + mints a DEK set with it. +*/}} +{{- if and (ne (.Values.encryption.fields | default "") "disabled") (not .Values.encryption.deksetSecretName) }} +{{- fail "encryption.deksetSecretName is required unless encryption.fields is \"disabled\": without a persistent DEK set the proxy mints a new key on every start and cannot decrypt data written before a restart. Mint one with the proxy's `backfill generate-dekset`, store it in a Secret and set encryption.deksetSecretName" }} +{{- end }} {{- if eq $backend "static" }} {{- /* Static KMS: a self-generated 32-byte KEK kept in a Secret. Generated once diff --git a/packages/system/keycloak/tests/encryption_test.yaml b/packages/system/keycloak/tests/encryption_test.yaml index 6b0a2a63d..4e5e09416 100644 --- a/packages/system/keycloak/tests/encryption_test.yaml +++ b/packages/system/keycloak/tests/encryption_test.yaml @@ -41,6 +41,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -55,6 +56,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -83,6 +85,7 @@ tests: value: Secret set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -180,6 +183,7 @@ tests: value: Service set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -198,6 +202,7 @@ tests: value: StatefulSet set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -220,6 +225,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -245,6 +251,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -261,6 +268,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault _cluster: root-host: example.org @@ -273,6 +281,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.keyName: keycloak encryption.kms.vault.tokenSecretName: keycloak-vault-token @@ -287,6 +296,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.tokenSecretName: keycloak-vault-token @@ -301,6 +311,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -318,6 +329,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -361,6 +373,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -379,6 +392,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -414,6 +428,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -429,6 +444,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -447,6 +463,7 @@ tests: value: ServiceAccount set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -465,6 +482,7 @@ tests: value: ServiceAccount set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -485,6 +503,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset _cluster: root-host: example.org cluster-domain: cozy.local @@ -503,6 +522,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.vault.svc:8200 encryption.kms.vault.keyName: keycloak @@ -516,13 +536,39 @@ tests: path: spec.template.spec.automountServiceAccountToken value: true - - it: mints an ephemeral DEK (no dekset) by default + - it: refuses to encrypt without a persistent DEK set + # Without KKP_DEKSET_FILE the proxy mints a new DEK on every start and + # cannot decrypt rows written before a restart. + template: templates/proxy.yaml + set: + encryption.enabled: true + _cluster: + root-host: example.org + cluster-domain: cozy.local + asserts: + - failedTemplate: + errorPattern: "encryption.deksetSecretName is required unless encryption.fields is" + + - it: refuses to encrypt only emails without a persistent DEK set + template: templates/proxy.yaml + set: + encryption.enabled: true + encryption.fields: email-only + _cluster: + root-host: example.org + cluster-domain: cozy.local + asserts: + - failedTemplate: + errorPattern: "encryption.deksetSecretName is required unless encryption.fields is" + + - it: runs as a passthrough without a DEK set when fields are disabled template: templates/proxy.yaml documentSelector: path: kind value: Deployment set: encryption.enabled: true + encryption.fields: disabled _cluster: root-host: example.org cluster-domain: cozy.local @@ -534,6 +580,11 @@ tests: - equal: path: spec.template.spec.containers[0].name value: proxy + - contains: + path: spec.template.spec.containers[0].env + content: + name: KKP_FIELDS + value: disabled - notContains: path: spec.template.spec.containers[0].env any: true @@ -585,6 +636,7 @@ tests: set: encryption.enabled: true encryption.replicas: 2 + encryption.fields: disabled _cluster: root-host: example.org cluster-domain: cozy.local @@ -631,6 +683,7 @@ tests: - it: trusts a private Vault CA supplied inline as caBundle set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak @@ -705,6 +758,7 @@ tests: - it: mounts a pre-seeded CA Secret when caSecretName is set set: encryption.enabled: true + encryption.fields: disabled encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak @@ -763,6 +817,7 @@ tests: - it: fails when both caBundle and caSecretName are set set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak @@ -781,6 +836,7 @@ tests: - it: leaves the proxy on the system trust store when no CA is configured set: encryption.enabled: true + encryption.fields: disabled encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak @@ -811,6 +867,7 @@ tests: - it: ignores a pre-seeded Vault CA Secret under the static backend set: encryption.enabled: true + encryption.fields: disabled encryption.kms.vault.caSecretName: vault-ca _cluster: root-host: example.org @@ -833,6 +890,7 @@ tests: - it: ignores an inline Vault CA bundle under the static backend set: encryption.enabled: true + encryption.fields: disabled encryption.kms.vault.caBundle: PEM _cluster: root-host: example.org @@ -854,6 +912,7 @@ tests: - it: does not enforce Vault CA mutual exclusion under the static backend set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.vault.caSecretName: vault-ca encryption.kms.vault.caBundle: PEM _cluster: @@ -868,6 +927,7 @@ tests: - it: rolls the pods when a chart-owned caBundle is rotated set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak @@ -893,6 +953,7 @@ tests: - it: rejects a caBundle that is not a PEM certificate bundle set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak @@ -910,6 +971,7 @@ tests: - it: reads a pre-seeded bundle from a custom caSecretKey set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.kms.backend: vault-transit encryption.kms.vault.address: https://vault.example encryption.kms.vault.keyName: keycloak diff --git a/packages/system/keycloak/tests/placement_test.yaml b/packages/system/keycloak/tests/placement_test.yaml index d3ddad482..ea8b0a3c6 100644 --- a/packages/system/keycloak/tests/placement_test.yaml +++ b/packages/system/keycloak/tests/placement_test.yaml @@ -61,6 +61,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset asserts: - notExists: path: spec.template.spec.affinity @@ -74,6 +75,7 @@ tests: value: Deployment set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset encryption.affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -104,6 +106,7 @@ tests: template: templates/proxy.yaml set: encryption.enabled: true + encryption.deksetSecretName: keycloak-kms-proxy-dekset asserts: - hasDocuments: count: 4 diff --git a/packages/system/keycloak/values.yaml b/packages/system/keycloak/values.yaml index 282707dfa..2b54d0da5 100644 --- a/packages/system/keycloak/values.yaml +++ b/packages/system/keycloak/values.yaml @@ -111,11 +111,17 @@ extraEnv: [] # Project: https://github.com/cozystack/keycloak-kms-proxy # # OPERATIONAL CAVEATS — read before enabling: -# * Availability downgrade. The proxy runs as a single replica (HA needs a -# shared wrapped DEK set, tracked as a follow-up). Keycloak itself runs HA -# (2 replicas); with encryption on, both replicas funnel their DB traffic -# through this one pod, so any proxy restart — node drain, rollout, image -# pull, OOM — briefly drops the database connection for the entire IdP. +# * Availability downgrade. The proxy runs as a single replica by default. +# Keycloak itself runs HA (2 replicas); with encryption on, both replicas +# funnel their DB traffic through this one pod, so any proxy restart — node +# drain, rollout, image pull, OOM — briefly drops the database connection +# for the entire IdP. Set encryption.replicas to run more than one. +# * A persistent DEK set is required. Unless fields is "disabled", the chart +# refuses to render without deksetSecretName: without it the proxy would +# mint a new DEK on every start and lose access to everything it encrypted +# before a restart. With kms.backend=static, deploy once with +# fields: disabled (passthrough) so the KEK Secret exists, mint the DEK set +# with that KEK, then set deksetSecretName and switch fields on. # * Existing data is NOT migrated automatically. Flipping enabled:true on a # populated database leaves the pre-existing PII as plaintext; logins that # match on the deterministic columns (username/email) then fail until the @@ -148,10 +154,10 @@ encryption: affinity: {} topologySpreadConstraints: [] # Existing Secret holding the wrapped DEK set (dekset.json) minted by the - # proxy's backfill tool (`cmd/backfill generate-dekset`). REQUIRED to migrate - # an existing populated database — the proxy must reuse the DEK the backfill - # encrypted rows with — and to run more than one replica. When empty the proxy - # mints an ephemeral DEK at startup, which is only safe on a fresh, empty DB. + # proxy's backfill tool (`cmd/backfill generate-dekset`). REQUIRED unless + # fields is "disabled": the proxy must load the same DEKs on every start to + # read what it encrypted before, and the backfill and every replica must + # share them. Back it up together with the KEK. deksetSecretName: "" deksetSecretKey: dekset.json kms: