mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 05:25:14 +09:00
[Backport release-1.4] fix(linstor): set controller resource requests and relax liveness probe (#3216)
## What this PR does Backport of #3043 to `release-1.4`. Under node CPU contention the `linstor-controller` JVM cannot answer its `GET /health` liveness probe within the piraeus-operator default timing (`timeoutSeconds: 1`, `failureThreshold: 3`). The kubelet then liveness-kills the container (exit 137), the controller crash-loops, its Service loses ready endpoints, `linstor-csi` gets `EPERM`, and RWX-Filesystem `DataVolume`s never bind — breaking tenant Kubernetes / NFS provisioning. On the `linstor-controller` container in `spec.controller.podTemplate`: - adds `resources.requests` (cpu `250m`, memory `700Mi`) — requests only, no limits: a CPU request guarantees CPU shares so the JVM keeps answering `/health` under contention, while omitting limits avoids throttling or OOM-killing the JVM; - relaxes the liveness timing to `timeoutSeconds: 5`, `failureThreshold: 6`. The change is confined to the `linstor` package (`cluster.yaml`, `values.yaml`) with a `helm unittest` suite (`tests/cluster_test.yaml`). The `linstor-csi` tag on `release-1.4` (`v1.10.6`) is preserved. ### Release note ```release-note fix(linstor): guarantee controller CPU/memory requests and relax the liveness probe so the linstor-controller does not crash-loop under node CPU contention ```
This commit is contained in:
3 files changed
+104
No files matched your search
@@ -37,6 +37,16 @@ spec:
|
||||
containers:
|
||||
- name: linstor-controller
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
# Guarantee CPU/memory so the JVM keeps answering its /health probe
|
||||
# under node contention. Requests only — see values.yaml controller.*.
|
||||
resources:
|
||||
{{- toYaml .Values.controller.resources | nindent 12 }}
|
||||
# Partial override: piraeus-operator merges podTemplate as a strategic
|
||||
# merge patch, so only these timing fields change. The operator's
|
||||
# httpGet (GET /health on the api port) is preserved as-is.
|
||||
livenessProbe:
|
||||
timeoutSeconds: {{ .Values.controller.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.controller.livenessProbe.failureThreshold }}
|
||||
- name: plunger
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
command:
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
suite: LinstorCluster controller resources and liveness hardening
|
||||
|
||||
# Pins the controller-pod surface this package adds on top of the
|
||||
# piraeus-operator defaults. The operator merges spec.controller.podTemplate
|
||||
# into the generated linstor-controller Deployment as a strategic merge patch
|
||||
# (containers merged by name, struct fields merged field-by-field), so:
|
||||
# - resources.requests guarantee the JVM gets scheduled CPU/memory so it can
|
||||
# answer GET /health under node CPU contention (no limits: avoid throttling
|
||||
# / OOM-killing the JVM);
|
||||
# - the livenessProbe override carries ONLY the timing fields (timeoutSeconds,
|
||||
# failureThreshold) relaxed from the operator defaults (1s / 3) so a JVM GC
|
||||
# pause or CPU contention does not trip a liveness kill. The httpGet target
|
||||
# (GET /health on the api port) is intentionally NOT overridden — it is
|
||||
# supplied by the operator, and re-stating it risks a wrong path/scheme.
|
||||
|
||||
templates:
|
||||
- templates/cluster.yaml
|
||||
|
||||
tests:
|
||||
- it: "linstor-controller container carries CPU and memory requests (no limits)"
|
||||
asserts:
|
||||
- isKind:
|
||||
of: LinstorCluster
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].name
|
||||
value: linstor-controller
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu
|
||||
value: 250m
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory
|
||||
value: 700Mi
|
||||
- notExists:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.limits
|
||||
|
||||
- it: "linstor-controller liveness probe relaxes timing but leaves httpGet to the operator"
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds
|
||||
value: 5
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold
|
||||
value: 6
|
||||
- notExists:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.httpGet
|
||||
|
||||
- it: "resource and liveness knobs are values-driven"
|
||||
set:
|
||||
controller:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 9
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu
|
||||
value: 500m
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory
|
||||
value: 1Gi
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds
|
||||
value: 10
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold
|
||||
value: 9
|
||||
|
||||
- it: "controller image pin is preserved alongside the new fields"
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.controller.podTemplate.spec.containers[0].image
|
||||
pattern: "^ghcr\\.io/cozystack/cozystack/piraeus-server:"
|
||||
@@ -14,3 +14,22 @@ linstorCSI:
|
||||
image:
|
||||
repository: ghcr.io/cozystack/cozystack/linstor-csi
|
||||
tag: v1.10.6@sha256:eb264710d3593778825a09a0cef94fb7774d72b55dd597a1e3282428990c42a2
|
||||
controller:
|
||||
# Resource requests for the linstor-controller JVM. Requests (not limits) are
|
||||
# set on purpose: a CPU request guarantees CPU shares so the JVM can answer its
|
||||
# GET /health liveness probe under node CPU contention, while omitting limits
|
||||
# avoids throttling the JVM (CPU limit) or OOM-killing it (memory limit). The
|
||||
# memory value is a conservative starting estimate for the JVM heap + overhead;
|
||||
# tune it to observed RSS.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 700Mi
|
||||
# Liveness-probe timing for the linstor-controller container. Only these timing
|
||||
# fields are overridden; the httpGet target (GET /health on the api port) is
|
||||
# supplied by piraeus-operator and intentionally left untouched. Relaxed from
|
||||
# the operator defaults (timeoutSeconds 1, failureThreshold 3) so a JVM GC
|
||||
# pause or CPU contention does not trip a kubelet liveness kill.
|
||||
livenessProbe:
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
Reference in new issue
Block a user