feat(keycloak): let Keycloak and its database proxy spread across zones

Neither the Keycloak StatefulSet nor the keycloak-kms-proxy Deployment
exposed any placement, so the scheduler was free to put both replicas of
each in one failure domain. For Keycloak that is worse than it looks: a
StatefulSet does not replace a pod stuck on an unreachable node, so losing
that domain takes every replica for as long as it stays lost.

Expose affinity and topologySpreadConstraints for both, empty by default
so nothing changes unless an installation opts in, and give the proxy a
PodDisruptionBudget once it runs more than one replica.

Assisted-By: LLM
Signed-off-by: Kirill Ilin <stitch14@yandex.ru>

packages/system/keycloak/templates/proxy.yaml

@@ -126,6 +126,14 @@ spec:
         app: keycloak-kms-proxy
     spec:
       serviceAccountName: keycloak-kms-proxy
+      {{- with .Values.encryption.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.encryption.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       automountServiceAccountToken: {{ $k8sAuth }}
       securityContext:
         runAsNonRoot: true
@@ -303,6 +311,20 @@ spec:
                 path: ca.crt
         {{- end }}
       {{- end }}
+{{- if gt $replicas 1 }}
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: keycloak-kms-proxy
+  labels:
+    app: keycloak-kms-proxy
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: keycloak-kms-proxy
+{{- end }}
 ---
 apiVersion: v1
 kind: Service

packages/system/keycloak/templates/sts.yaml

@@ -62,6 +62,14 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       securityContext:
         fsGroup: 1000
       {{- if $themes }}

packages/system/keycloak/tests/placement_test.yaml

@@ -0,0 +1,127 @@
+suite: keycloak and proxy placement across failure domains
+templates:
+  - templates/sts.yaml
+  - templates/proxy.yaml
+
+release:
+  name: keycloak
+  namespace: cozy-keycloak
+
+set:
+  _cluster:
+    root-host: example.org
+    cluster-domain: cozy.local
+
+tests:
+  - it: leaves Keycloak placement to the scheduler by default
+    template: templates/sts.yaml
+    documentSelector:
+      path: kind
+      value: StatefulSet
+    asserts:
+      - notExists:
+          path: spec.template.spec.affinity
+      - notExists:
+          path: spec.template.spec.topologySpreadConstraints
+
+  - it: renders Keycloak affinity and spread constraints when set
+    template: templates/sts.yaml
+    documentSelector:
+      path: kind
+      value: StatefulSet
+    set:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values: [zone-a, zone-b]
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: DoNotSchedule
+          nodeTaintsPolicy: Honor
+          labelSelector:
+            matchLabels:
+              app: keycloak-ha
+    asserts:
+      - equal:
+          path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values
+          value: [zone-a, zone-b]
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].nodeTaintsPolicy
+          value: Honor
+
+  - it: leaves proxy placement to the scheduler by default
+    template: templates/proxy.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    set:
+      encryption.enabled: true
+    asserts:
+      - notExists:
+          path: spec.template.spec.affinity
+      - notExists:
+          path: spec.template.spec.topologySpreadConstraints
+
+  - it: renders proxy affinity and spread constraints when set
+    template: templates/proxy.yaml
+    documentSelector:
+      path: kind
+      value: Deployment
+    set:
+      encryption.enabled: true
+      encryption.affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values: [zone-a, zone-b]
+      encryption.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: DoNotSchedule
+          nodeTaintsPolicy: Honor
+          labelSelector:
+            matchLabels:
+              app: keycloak-kms-proxy
+    asserts:
+      - equal:
+          path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values
+          value: [zone-a, zone-b]
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
+          value: DoNotSchedule
+
+  # A single replica has nothing to keep available during a drain, and a
+  # PodDisruptionBudget on it would only block the drain.
+  - it: renders no disruption budget for a single proxy replica
+    template: templates/proxy.yaml
+    set:
+      encryption.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 4
+
+  - it: keeps one proxy replica through voluntary disruptions when replicated
+    template: templates/proxy.yaml
+    documentSelector:
+      path: kind
+      value: PodDisruptionBudget
+    set:
+      encryption.enabled: true
+      encryption.replicas: 2
+      encryption.deksetSecretName: keycloak-kms-proxy-dekset
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 1
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app: keycloak-kms-proxy

packages/system/keycloak/values.yaml

@@ -66,6 +66,25 @@ themes: []
 # arrives via _cluster.branding, so no per-package values plumbing is required.
 
 imagePullSecrets: []
+
+# Placement of the Keycloak pods. Empty leaves it to the scheduler, which is
+# free to put both replicas in one failure domain; a StatefulSet does not
+# replace a pod stuck on an unreachable node, so that domain's loss then takes
+# every replica with it. Example spreading one replica per zone:
+#
+#   topologySpreadConstraints:
+#     - maxSkew: 1
+#       topologyKey: topology.kubernetes.io/zone
+#       whenUnsatisfiable: DoNotSchedule
+#       # Leave out nodes the pod cannot run on (an unreachable zone carries
+#       # the unreachable NoSchedule taint), so the constraint does not count
+#       # a lost zone as a domain the replacement pod must balance against.
+#       nodeTaintsPolicy: Honor
+#       labelSelector:
+#         matchLabels:
+#           app: keycloak-ha
+affinity: {}
+topologySpreadConstraints: []
 # - name: my-registry-secret
 
 # Extra environment variables to inject into the Keycloak container.
@@ -123,6 +142,11 @@ encryption:
   # Number of proxy replicas. >1 requires deksetSecretName (a shared wrapped
   # DEK set) so every replica encrypts deterministic columns under one key.
   replicas: 1
+  # Placement of the proxy pods, same shape as the top-level affinity and
+  # topologySpreadConstraints (the label is app: keycloak-kms-proxy). With more
+  # than one replica a PodDisruptionBudget keeps one of them up through drains.
+  affinity: {}
+  topologySpreadConstraints: []
   # Existing Secret holding the wrapped DEK set (dekset.json) minted by the
   # proxy's backfill tool (`cmd/backfill generate-dekset`). REQUIRED to migrate
   # an existing populated database — the proxy must reuse the DEK the backfill