From b8a8d64d90c99ba400bd79f36f0414f56ff31561 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 6 Jul 2026 18:01:49 +0300 Subject: [PATCH] fix(linstor): set controller resource requests and relax liveness probe Backport of #3043 to release-1.4. Under node CPU contention the linstor-controller JVM cannot answer its GET /health liveness probe within the piraeus-operator default timing (timeoutSeconds 1, failureThreshold 3). The kubelet then liveness-kills the container, the controller crash-loops, linstor-csi gets EPERM and RWX-Filesystem DataVolumes never bind. Add CPU/memory requests (requests only, no limits) so the JVM keeps answering /health under contention, and relax the liveness timing to timeoutSeconds 5 / failureThreshold 6. (cherry picked from commit 7801f7748e484e63cf93d6ecc5098596fb95fff0) Signed-off-by: IvanHunters --- .../system/linstor/templates/cluster.yaml | 10 +++ .../system/linstor/tests/cluster_test.yaml | 75 +++++++++++++++++++ packages/system/linstor/values.yaml | 19 +++++ 3 files changed, 104 insertions(+) create mode 100644 packages/system/linstor/tests/cluster_test.yaml diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml index e45b1dffb..32f53f4f4 100644 --- a/packages/system/linstor/templates/cluster.yaml +++ b/packages/system/linstor/templates/cluster.yaml @@ -37,6 +37,16 @@ spec: containers: - name: linstor-controller image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} + # Guarantee CPU/memory so the JVM keeps answering its /health probe + # under node contention. Requests only — see values.yaml controller.*. + resources: + {{- toYaml .Values.controller.resources | nindent 12 }} + # Partial override: piraeus-operator merges podTemplate as a strategic + # merge patch, so only these timing fields change. The operator's + # httpGet (GET /health on the api port) is preserved as-is. + livenessProbe: + timeoutSeconds: {{ .Values.controller.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.controller.livenessProbe.failureThreshold }} - name: plunger image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} command: diff --git a/packages/system/linstor/tests/cluster_test.yaml b/packages/system/linstor/tests/cluster_test.yaml new file mode 100644 index 000000000..4bf7ba094 --- /dev/null +++ b/packages/system/linstor/tests/cluster_test.yaml @@ -0,0 +1,75 @@ +suite: LinstorCluster controller resources and liveness hardening + +# Pins the controller-pod surface this package adds on top of the +# piraeus-operator defaults. The operator merges spec.controller.podTemplate +# into the generated linstor-controller Deployment as a strategic merge patch +# (containers merged by name, struct fields merged field-by-field), so: +# - resources.requests guarantee the JVM gets scheduled CPU/memory so it can +# answer GET /health under node CPU contention (no limits: avoid throttling +# / OOM-killing the JVM); +# - the livenessProbe override carries ONLY the timing fields (timeoutSeconds, +# failureThreshold) relaxed from the operator defaults (1s / 3) so a JVM GC +# pause or CPU contention does not trip a liveness kill. The httpGet target +# (GET /health on the api port) is intentionally NOT overridden — it is +# supplied by the operator, and re-stating it risks a wrong path/scheme. + +templates: + - templates/cluster.yaml + +tests: + - it: "linstor-controller container carries CPU and memory requests (no limits)" + asserts: + - isKind: + of: LinstorCluster + - equal: + path: spec.controller.podTemplate.spec.containers[0].name + value: linstor-controller + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu + value: 250m + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory + value: 700Mi + - notExists: + path: spec.controller.podTemplate.spec.containers[0].resources.limits + + - it: "linstor-controller liveness probe relaxes timing but leaves httpGet to the operator" + asserts: + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds + value: 5 + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold + value: 6 + - notExists: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.httpGet + + - it: "resource and liveness knobs are values-driven" + set: + controller: + resources: + requests: + cpu: 500m + memory: 1Gi + livenessProbe: + timeoutSeconds: 10 + failureThreshold: 9 + asserts: + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu + value: 500m + - equal: + path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory + value: 1Gi + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds + value: 10 + - equal: + path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold + value: 9 + + - it: "controller image pin is preserved alongside the new fields" + asserts: + - matchRegex: + path: spec.controller.podTemplate.spec.containers[0].image + pattern: "^ghcr\\.io/cozystack/cozystack/piraeus-server:" diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index f2192c5ed..76fd0202f 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -14,3 +14,22 @@ linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi tag: v1.10.6@sha256:eb264710d3593778825a09a0cef94fb7774d72b55dd597a1e3282428990c42a2 +controller: + # Resource requests for the linstor-controller JVM. Requests (not limits) are + # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its + # GET /health liveness probe under node CPU contention, while omitting limits + # avoids throttling the JVM (CPU limit) or OOM-killing it (memory limit). The + # memory value is a conservative starting estimate for the JVM heap + overhead; + # tune it to observed RSS. + resources: + requests: + cpu: 250m + memory: 700Mi + # Liveness-probe timing for the linstor-controller container. Only these timing + # fields are overridden; the httpGet target (GET /health on the api port) is + # supplied by piraeus-operator and intentionally left untouched. Relaxed from + # the operator defaults (timeoutSeconds 1, failureThreshold 3) so a JVM GC + # pause or CPU contention does not trip a kubelet liveness kill. + livenessProbe: + timeoutSeconds: 5 + failureThreshold: 6