From 52c20ad1a2dfd3553449c5cb0a97f014744b52bd Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 23:13:05 +0300 Subject: [PATCH] fix(linstor): set controller resource requests and relax liveness probe Under node CPU contention the linstor-controller JVM cannot answer its GET /health liveness probe within the piraeus-operator defaults (timeoutSeconds 1, failureThreshold 3). The kubelet then liveness-kills the container; the resulting crash-loop leaves the linstor-controller Service with no ready endpoints, linstor-csi gets EPERM, and RWX-Filesystem DataVolumes never bind, breaking tenant provisioning. The operator-generated controller pod sets no resources (no CPU request, hence no scheduling/CPU guarantee) and ships the tight default probe. Add CPU/memory requests and relax the liveness timing via the controller podTemplate, which piraeus-operator merges into the Deployment as a strategic merge patch. The probe override carries only the timing fields; the httpGet target (GET /health on the api port) is left to the operator to avoid pinning a wrong path or scheme. Requests-only (no limits) keeps the JVM from being throttled or OOM-killed. The knobs are exposed under controller.* for tuning to observed RSS. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- .../system/linstor/templates/cluster.yaml | 10 +++ .../system/linstor/tests/cluster_test.yaml | 75 +++++++++++++++++++ packages/system/linstor/values.yaml | 19 +++++ 3 files changed, 104 insertions(+) create mode 100644 packages/system/linstor/tests/cluster_test.yaml diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml index ab349cf5d..4e94c80f9 100644 --- a/packages/system/linstor/templates/cluster.yaml +++ b/packages/system/linstor/templates/cluster.yaml @@ -39,6 +39,16 @@ spec: containers: - name: linstor-controller image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} + # Guarantee CPU/memory so the JVM keeps answering its /health probe + # under node contention. Requests only — see values.yaml controller.*. + resources: + {{- toYaml .Values.controller.resources | nindent 12 }} + # Partial override: piraeus-operator merges podTemplate as a strategic + # merge patch, so only these timing fields change. The operator's + # httpGet (GET /health on the api port) is preserved as-is. + livenessProbe: + timeoutSeconds: {{ .Values.controller.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.controller.livenessProbe.failureThreshold }} - name: plunger image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} command: diff --git a/packages/system/linstor/tests/cluster_test.yaml b/packages/system/linstor/tests/cluster_test.yaml new file mode 100644 index 000000000..4bf7ba094 --- /dev/null +++ b/packages/system/linstor/tests/cluster_test.yaml @@ -0,0 +1,75 @@ +suite: LinstorCluster controller resources and liveness hardening + +# Pins the controller-pod surface this package adds on top of the +# piraeus-operator defaults. The operator merges spec.controller.podTemplate +# into the generated linstor-controller Deployment as a strategic merge patch +# (containers merged by name, struct fields merged field-by-field), so: +# - resources.requests guarantee the JVM gets scheduled CPU/memory so it can +# answer GET /health under node CPU contention (no limits: avoid throttling +# / OOM-killing the JVM); +# - the livenessProbe override carries ONLY the timing fields (timeoutSeconds, +# failureThreshold) relaxed from the operator defaults (1s / 3) so a JVM GC +# pause or CPU contention does not trip a liveness kill. The httpGet target +# (GET /health on the api port) is intentionally NOT overridden — it is +# supplied by the operator, and re-stating it risks a wrong path/scheme. + +templates: + - templates/cluster.yaml + +tests: + - it: "linstor-controller container carries CPU and memory requests (no limits)" + asserts: + - isKind: + of: LinstorCluster + - equal: + path: spec.controller.podTemplate.spec.containers[0].name + value: linstor-controller + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu + value: 250m + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory + value: 700Mi + - notExists: + path: spec.controller.podTemplate.spec.containers[0].resources.limits + + - it: "linstor-controller liveness probe relaxes timing but leaves httpGet to the operator" + asserts: + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds + value: 5 + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold + value: 6 + - notExists: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.httpGet + + - it: "resource and liveness knobs are values-driven" + set: + controller: + resources: + requests: + cpu: 500m + memory: 1Gi + livenessProbe: + timeoutSeconds: 10 + failureThreshold: 9 + asserts: + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu + value: 500m + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory + value: 1Gi + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds + value: 10 + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold + value: 9 + + - it: "controller image pin is preserved alongside the new fields" + asserts: + - matchRegex: + path: spec.controller.podTemplate.spec.containers[0].image + pattern: "^ghcr\\.io/cozystack/cozystack/piraeus-server:" diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 09cf14cda..6ff6b2382 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -14,3 +14,22 @@ linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi tag: v1.5.0@sha256:c60519d12aa8d078e4891d6e436c709ff53f2cc96457994390ca9424b6505903 +controller: + # Resource requests for the linstor-controller JVM. Requests (not limits) are + # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its + # GET /health liveness probe under node CPU contention, while omitting limits + # avoids throttling the JVM (CPU limit) or OOM-killing it (memory limit). The + # memory value is a conservative starting estimate for the JVM heap + overhead; + # tune it to observed RSS. + resources: + requests: + cpu: 250m + memory: 700Mi + # Liveness-probe timing for the linstor-controller container. Only these timing + # fields are overridden; the httpGet target (GET /health on the api port) is + # supplied by piraeus-operator and intentionally left untouched. Relaxed from + # the operator defaults (timeoutSeconds 1, failureThreshold 3) so a JVM GC + # pause or CPU contention does not trip a kubelet liveness kill. + livenessProbe: + timeoutSeconds: 5 + failureThreshold: 6