mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
chore(kafka, kafka-operator): drop the JVM sizing workaround
The workaround passed MaxRAM and ActiveProcessorCount to every Strimzi JVM because the JDK 17 in the 0.45 images could not see the container limits on kernels without CONFIG_MEMCG_V1 (JDK-8347811). The Strimzi 1.2.0 images ship JDK 21.0.12, which carries the fix. On such a kernel, with a 1Gi limit, Strimzi's heap probe in the Kafka image now estimates a 495M maximum heap, where the 0.45.1-rc1 image estimated 13.9G. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
7 files changed
+1
-174
No files matched your search
@@ -1,20 +0,0 @@
|
||||
{{- /*
|
||||
JAVA_TOOL_OPTIONS for a Strimzi-managed container, from its resource limits.
|
||||
The JDK 17 in the Strimzi images cannot see the container limits on kernels
|
||||
built without CONFIG_MEMCG_V1 (JDK-8347811) and sizes itself from the host:
|
||||
Strimzi's heap probe then hands every broker up to a 5Gi heap. Every java in
|
||||
the container reads this variable, the probe included, so Strimzi's own
|
||||
heap rules work again. jvmOptions -XX would not do: Strimzi moves those into
|
||||
KAFKA_JVM_PERFORMANCE_OPTS, and kafka-run-class.sh drops its G1 tuning when
|
||||
that variable is set. ActiveProcessorCount only when the chart sets a CPU
|
||||
limit: a default a LimitRange adds at admission is not visible here, and
|
||||
Strimzi template env takes no resourceFieldRef, so such a JVM still counts
|
||||
the host CPUs.
|
||||
*/ -}}
|
||||
{{- define "kafka.javaToolOptions" -}}
|
||||
{{- $opts := list (printf "-XX:MaxRAM=%d" (include "cozy-lib.resources.toFloat" .memory | float64 | int64)) -}}
|
||||
{{- with .cpu -}}
|
||||
{{- $opts = append $opts (printf "-XX:ActiveProcessorCount=%d" (include "cozy-lib.resources.toFloat" . | float64 | ceil | int64)) -}}
|
||||
{{- end -}}
|
||||
{{- join " " $opts -}}
|
||||
{{- end -}}
|
||||
@@ -80,8 +80,7 @@ spec:
|
||||
# The topic-operator and user-operator are JVM processes and OOMKill at
|
||||
# that default, leaving the entity-operator in CrashLoopBackOff so
|
||||
# KafkaTopics/KafkaUsers never reconcile. Set explicit resources so they
|
||||
# don't inherit the namespace default. JAVA_TOOL_OPTIONS: see
|
||||
# kafka.javaToolOptions.
|
||||
# don't inherit the namespace default.
|
||||
topicOperator:
|
||||
resources:
|
||||
requests:
|
||||
@@ -107,11 +106,3 @@ spec:
|
||||
labels:
|
||||
policy.cozystack.io/allow-to-apiserver: "true"
|
||||
enableServiceLinks: false
|
||||
topicOperatorContainer:
|
||||
env:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: {{ include "kafka.javaToolOptions" (dict "memory" $entityOperatorMemory) | quote }}
|
||||
userOperatorContainer:
|
||||
env:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: {{ include "kafka.javaToolOptions" (dict "memory" $entityOperatorMemory) | quote }}
|
||||
@@ -107,12 +107,6 @@ spec:
|
||||
roles:
|
||||
- broker
|
||||
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.kafka.resourcesPreset .Values.kafka.resources $) | nindent 18 }}
|
||||
# JAVA_TOOL_OPTIONS: see kafka.javaToolOptions.
|
||||
template:
|
||||
kafkaContainer:
|
||||
env:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: {{ include "kafka.javaToolOptions" (include "cozy-lib.resources.defaultingSanitize" (list .Values.kafka.resourcesPreset .Values.kafka.resources $) | fromYaml).limits | quote }}
|
||||
storage:
|
||||
type: jbod
|
||||
volumes:
|
||||
@@ -152,11 +146,6 @@ spec:
|
||||
roles:
|
||||
- controller
|
||||
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.kafka.resourcesPreset .Values.kafka.resources $) | nindent 18 }}
|
||||
template:
|
||||
kafkaContainer:
|
||||
env:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: {{ include "kafka.javaToolOptions" (include "cozy-lib.resources.defaultingSanitize" (list .Values.kafka.resourcesPreset .Values.kafka.resources $) | fromYaml).limits | quote }}
|
||||
storage:
|
||||
type: jbod
|
||||
volumes:
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
suite: Kafka JVMs are sized from their container limits
|
||||
|
||||
templates:
|
||||
- templates/migration-hook.yaml
|
||||
- templates/kafka.yaml
|
||||
|
||||
release:
|
||||
name: test-kafka
|
||||
namespace: tenant-test
|
||||
|
||||
tests:
|
||||
- it: passes the default preset limits to every node pool JVM
|
||||
template: templates/migration-hook.yaml
|
||||
documentIndex: 3
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].args[0]
|
||||
pattern: "release-namespace: \\$\\{ns\\}\\s+spec:\\s+replicas: \\d+\\s+roles:\\s+- broker\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=1073741824 -XX:ActiveProcessorCount=1\"\\s+storage:"
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].args[0]
|
||||
pattern: "roles:\\s+- controller\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=1073741824 -XX:ActiveProcessorCount=1\"\\s+storage:"
|
||||
|
||||
# Strimzi moves jvmOptions -XX into KAFKA_JVM_PERFORMANCE_OPTS, and
|
||||
# kafka-run-class.sh then drops its G1 settings.
|
||||
- it: leaves the node pool JVM options to Strimzi and Kafka
|
||||
template: templates/migration-hook.yaml
|
||||
documentSelector:
|
||||
path: kind
|
||||
value: Job
|
||||
asserts:
|
||||
- notMatchRegex:
|
||||
path: spec.template.spec.containers[0].args[0]
|
||||
pattern: "jvmOptions"
|
||||
|
||||
- it: rounds a fractional CPU limit up
|
||||
template: templates/migration-hook.yaml
|
||||
documentIndex: 3
|
||||
set:
|
||||
kafka.resources:
|
||||
cpu: 2500m
|
||||
memory: 3Gi
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].args[0]
|
||||
pattern: "roles:\\s+- controller\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=3221225472 -XX:ActiveProcessorCount=3\"\\s+storage:"
|
||||
|
||||
- it: takes the limits, not the requests, when the allocation ratios shrink the requests
|
||||
template: templates/migration-hook.yaml
|
||||
documentIndex: 3
|
||||
set:
|
||||
_cluster:
|
||||
cpu-allocation-ratio: "4"
|
||||
memory-allocation-ratio: "2"
|
||||
kafka.resources:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].args[0]
|
||||
pattern: "release-namespace: \\$\\{ns\\}\\s+spec:\\s+replicas: \\d+\\s+roles:\\s+- broker\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=2147483648 -XX:ActiveProcessorCount=2\"\\s+storage:"
|
||||
- matchRegex:
|
||||
path: spec.template.spec.containers[0].args[0]
|
||||
pattern: "roles:\\s+- controller\\s+resources:\\s+limits:(\\s+\\S+: \\S+)+\\s+requests:(\\s+\\S+: \\S+)+(\\s+#[^\\n]*)*\\s+template:\\s+kafkaContainer:\\s+env:\\s+- name: JAVA_TOOL_OPTIONS\\s+value: \"-XX:MaxRAM=2147483648 -XX:ActiveProcessorCount=2\"\\s+storage:"
|
||||
|
||||
- it: passes the entity operator memory limit to its JVMs
|
||||
template: templates/kafka.yaml
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.entityOperator.topicOperator.resources.limits.memory
|
||||
value: 512Mi
|
||||
- equal:
|
||||
path: spec.entityOperator.template.topicOperatorContainer.env
|
||||
value:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: -XX:MaxRAM=536870912
|
||||
- equal:
|
||||
path: spec.entityOperator.userOperator.resources.limits.memory
|
||||
value: 512Mi
|
||||
- equal:
|
||||
path: spec.entityOperator.template.userOperatorContainer.env
|
||||
value:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: -XX:MaxRAM=536870912
|
||||
@@ -3,9 +3,6 @@ export NAMESPACE=cozy-$(NAME)
|
||||
|
||||
include ../../../hack/package.mk
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
|
||||
update:
|
||||
rm -rf charts
|
||||
helm pull oci://quay.io/strimzi-helm/strimzi-kafka-operator --untar --untardir charts --version 1.2.0
|
||||
@@ -1,31 +0,0 @@
|
||||
suite: cluster operator JVM is sized from the container limits
|
||||
release:
|
||||
name: kafka-operator
|
||||
namespace: cozy-kafka-operator
|
||||
templates:
|
||||
- charts/strimzi-kafka-operator/templates/060-Deployment-strimzi-cluster-operator.yaml
|
||||
tests:
|
||||
- it: passes the container limits to the JVM, which cannot read them on kernels without cgroup v1 controllers
|
||||
asserts:
|
||||
# $(VAR) expands only from entries earlier in the list.
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].env[19]
|
||||
value:
|
||||
name: CONTAINER_MEMORY_LIMIT
|
||||
valueFrom:
|
||||
resourceFieldRef:
|
||||
resource: limits.memory
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].env[20]
|
||||
value:
|
||||
name: CONTAINER_CPU_LIMIT
|
||||
valueFrom:
|
||||
resourceFieldRef:
|
||||
resource: limits.cpu
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].env[21]
|
||||
value:
|
||||
name: JAVA_OPTS
|
||||
value: -XX:MaxRAM=$(CONTAINER_MEMORY_LIMIT) -XX:ActiveProcessorCount=$(CONTAINER_CPU_LIMIT)
|
||||
- exists:
|
||||
path: spec.template.spec.containers[0].resources.limits.cpu
|
||||
@@ -5,19 +5,3 @@ strimzi-kafka-operator:
|
||||
resources:
|
||||
limits:
|
||||
memory: 512Mi
|
||||
# The JDK 17 in the operator image decides whether to read the container
|
||||
# limits by looking at /proc/cgroups, which loses its memory line on kernels
|
||||
# built without CONFIG_MEMCG_V1 (6.12+). The JVM then sizes its heap from
|
||||
# host RAM and the container is OOMKilled. Drop this once the image ships a
|
||||
# JDK with JDK-8347811 (17.0.20, 21.0.10).
|
||||
extraEnvs:
|
||||
- name: CONTAINER_MEMORY_LIMIT
|
||||
valueFrom:
|
||||
resourceFieldRef:
|
||||
resource: limits.memory
|
||||
- name: CONTAINER_CPU_LIMIT
|
||||
valueFrom:
|
||||
resourceFieldRef:
|
||||
resource: limits.cpu
|
||||
- name: JAVA_OPTS
|
||||
value: -XX:MaxRAM=$(CONTAINER_MEMORY_LIMIT) -XX:ActiveProcessorCount=$(CONTAINER_CPU_LIMIT)
|
||||
Reference in new issue
Block a user