mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-06 12:03:39 +09:00
fix(linstor): set controller resource requests and relax liveness probe
Backport of #3043 to release-1.4.
Under node CPU contention the linstor-controller JVM cannot answer its
GET /health liveness probe within the piraeus-operator default timing
(timeoutSeconds 1, failureThreshold 3). The kubelet then liveness-kills
the container, the controller crash-loops, linstor-csi gets EPERM and
RWX-Filesystem DataVolumes never bind. Add CPU/memory requests (requests
only, no limits) so the JVM keeps answering /health under contention, and
relax the liveness timing to timeoutSeconds 5 / failureThreshold 6.
(cherry picked from commit 7801f7748e)
Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
This commit is contained in:
3 files changed
+104
No files matched your search
@@ -37,6 +37,16 @@ spec:
|
||||
containers:
|
||||
- name: linstor-controller
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
# Guarantee CPU/memory so the JVM keeps answering its /health probe
|
||||
# under node contention. Requests only — see values.yaml controller.*.
|
||||
resources:
|
||||
{{- toYaml .Values.controller.resources | nindent 12 }}
|
||||
# Partial override: piraeus-operator merges podTemplate as a strategic
|
||||
# merge patch, so only these timing fields change. The operator's
|
||||
# httpGet (GET /health on the api port) is preserved as-is.
|
||||
livenessProbe:
|
||||
timeoutSeconds: {{ .Values.controller.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.controller.livenessProbe.failureThreshold }}
|
||||
- name: plunger
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
command:
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
suite: LinstorCluster controller resources and liveness hardening
|
||||
|
||||
# Pins the controller-pod surface this package adds on top of the
|
||||
# piraeus-operator defaults. The operator merges spec.controller.podTemplate
|
||||
# into the generated linstor-controller Deployment as a strategic merge patch
|
||||
# (containers merged by name, struct fields merged field-by-field), so:
|
||||
# - resources.requests guarantee the JVM gets scheduled CPU/memory so it can
|
||||
# answer GET /health under node CPU contention (no limits: avoid throttling
|
||||
# / OOM-killing the JVM);
|
||||
# - the livenessProbe override carries ONLY the timing fields (timeoutSeconds,
|
||||
# failureThreshold) relaxed from the operator defaults (1s / 3) so a JVM GC
|
||||
# pause or CPU contention does not trip a liveness kill. The httpGet target
|
||||
# (GET /health on the api port) is intentionally NOT overridden — it is
|
||||
# supplied by the operator, and re-stating it risks a wrong path/scheme.
|
||||
|
||||
templates:
|
||||
- templates/cluster.yaml
|
||||
|
||||
tests:
|
||||
- it: "linstor-controller container carries CPU and memory requests (no limits)"
|
||||
asserts:
|
||||
- isKind:
|
||||
of: LinstorCluster
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].name
|
||||
value: linstor-controller
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu
|
||||
value: 250m
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory
|
||||
value: 700Mi
|
||||
- notExists:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.limits
|
||||
|
||||
- it: "linstor-controller liveness probe relaxes timing but leaves httpGet to the operator"
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds
|
||||
value: 5
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold
|
||||
value: 6
|
||||
- notExists:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.httpGet
|
||||
|
||||
- it: "resource and liveness knobs are values-driven"
|
||||
set:
|
||||
controller:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
livenessProbe:
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 9
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.cpu
|
||||
value: 500m
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].resources.requests.memory
|
||||
value: 1Gi
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.timeoutSeconds
|
||||
value: 10
|
||||
- equal:
|
||||
path: spec.controller.podTemplate.spec.containers[0].livenessProbe.failureThreshold
|
||||
value: 9
|
||||
|
||||
- it: "controller image pin is preserved alongside the new fields"
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.controller.podTemplate.spec.containers[0].image
|
||||
pattern: "^ghcr\\.io/cozystack/cozystack/piraeus-server:"
|
||||
@@ -14,3 +14,22 @@ linstorCSI:
|
||||
image:
|
||||
repository: ghcr.io/cozystack/cozystack/linstor-csi
|
||||
tag: v1.10.6@sha256:eb264710d3593778825a09a0cef94fb7774d72b55dd597a1e3282428990c42a2
|
||||
controller:
|
||||
# Resource requests for the linstor-controller JVM. Requests (not limits) are
|
||||
# set on purpose: a CPU request guarantees CPU shares so the JVM can answer its
|
||||
# GET /health liveness probe under node CPU contention, while omitting limits
|
||||
# avoids throttling the JVM (CPU limit) or OOM-killing it (memory limit). The
|
||||
# memory value is a conservative starting estimate for the JVM heap + overhead;
|
||||
# tune it to observed RSS.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 700Mi
|
||||
# Liveness-probe timing for the linstor-controller container. Only these timing
|
||||
# fields are overridden; the httpGet target (GET /health on the api port) is
|
||||
# supplied by piraeus-operator and intentionally left untouched. Relaxed from
|
||||
# the operator defaults (timeoutSeconds 1, failureThreshold 3) so a JVM GC
|
||||
# pause or CPU contention does not trip a kubelet liveness kill.
|
||||
livenessProbe:
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
Reference in new issue
Block a user