diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 92b40ef4d..d69a3e493 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -248,13 +248,19 @@ spec: {{- /* An explicit null is dropped before schema validation, so neither bound sees it, and the master exits on the empty flag it renders. */}} volumeSizeLimitMB: {{ required "master.volumeSizeLimitMB must be set" .Values.master.volumeSizeLimitMB }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | nindent 10 }} +{{- $masterResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | fromYaml }} + resources: {{- toYaml $masterResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $masterResources | quote }} volume: {{ if eq .Values.topology "MultiZone" }} enabled: false {{- end }} replicas: {{ .Values.volume.replicas }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | nindent 10 }} +{{- $volumeResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | fromYaml }} + resources: {{- toYaml $volumeResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -273,7 +279,13 @@ spec: {{- range $poolName, $pool := .Values.volume.pools }} {{ $poolName }}: replicas: {{ ternary $pool.replicas $.Values.volume.replicas (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -289,7 +301,9 @@ spec: {{- range $zoneName, $zone := .Values.volume.zones }} {{ $zoneName }}: replicas: {{ ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list $.Values.volume.resourcesPreset $.Values.volume.resources $) | nindent 12 }} + resources: {{- toYaml $volumeResources | nindent 12 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -314,7 +328,13 @@ spec: {{- range $poolName, $pool := (dig "pools" dict $zone) }} {{ $zoneName }}-{{ $poolName }}: replicas: {{ ternary $pool.replicas (ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas")) (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -345,7 +365,10 @@ spec: replicas: {{ .Values.filer.replicas }} s3: domainName: {{ .Values.host | default (printf "s3.%s" $host) }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | nindent 10 }} +{{- $filerResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | fromYaml }} + resources: {{- toYaml $filerResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $filerResources | quote }} s3: replicas: {{ .Values.s3.replicas }} ingress: @@ -381,7 +404,10 @@ spec: {{- if not $wildcardSecret }} secretName: {{ .Release.Name }}-s3-ingress-tls {{- end }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }} +{{- $s3Resources := include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | fromYaml }} + resources: {{- toYaml $s3Resources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $s3Resources | quote }} admin: enabled: {{ .Values.lifecycle.enabled }} secret: diff --git a/packages/extra/seaweedfs/tests/gomemlimit_test.yaml b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml new file mode 100644 index 000000000..2b298b10e --- /dev/null +++ b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml @@ -0,0 +1,162 @@ +suite: seaweedfs components carry a GOMEMLIMIT derived from their memory limit +templates: + - templates/seaweedfs.yaml + +# Every SeaweedFS component is a Go binary. Without GOMEMLIMIT the runtime +# collects on a relative target and keeps no absolute ceiling, so a burst of +# allocation grows the heap past the cgroup limit and the kernel kills the +# container. Each component sizes its own value from its own memory limit, +# so the assertions below track the limit rather than a fixed string. + +tests: + - it: derives GOMEMLIMIT from the default preset on every component + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + asserts: + - equal: + path: spec.values.seaweedfs.master.resources.limits.memory + value: 512Mi + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.filer.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: follows the memory limit when a component is resized + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 8Gi + volume: + resources: + memory: 2Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 8Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 6144MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: keeps GOMEMLIMIT below the cgroup limit it protects + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 1Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 1Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 768MiB + + # The subchart merges volume into every entry of volumes, so a pool that + # did not carry its own value would run on the default volume's, sized for + # a larger limit than its own. + - it: sizes a Simple pool from its own limit, not the default volume's + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: sizes zones and zone pools from their own limits in MultiZone + documentSelector: + path: kind + value: HelmRelease + set: + topology: MultiZone + replicationFactor: 1 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + resources: + memory: 2Gi + zones: + a: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.volumes.a-small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.a-small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 48b0718fd..5830e1400 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -202,6 +202,26 @@ {{- printf `-Xms%dm -Xmx%dm` $xmsMi $xmxMi }} {{- end }} +{{- /* + goMemLimit takes a sanitized .Values.resources and returns a GOMEMLIMIT + value set to 75% of the memory limit. Without it the Go runtime collects + on a relative target (GOGC) with no absolute ceiling, so a burst of + allocation grows the heap past the cgroup limit and the kernel kills the + container. The 25% headroom covers what GOMEMLIMIT does not account for: + goroutine stacks and any mmap'd region. Accepts only sanitized resource + maps. +*/}} +{{- define "cozy-lib.resources.goMemLimit" }} +{{- $memoryLimitInt := include "cozy-lib.resources.toFloat" .limits.memory | float64 | int64 }} +{{- /* 4194304 is 4Mi */}} +{{- $limitMiB := div (mul $memoryLimitInt 3) 4194304 }} +{{- /* Go accepts 0, and then collects almost without pause. */}} +{{- if lt $limitMiB 1 }} +{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs a limit of at least 1398102 bytes (4/3 Mi) to come out at 1MiB." .limits.memory) }} +{{- end }} +{{- printf `%dMiB` $limitMiB }} +{{- end }} + {{- define "cozy-lib.resources.flatten" -}} {{- $out := dict -}} {{- $res := include "cozy-lib.resources.sanitize" . | fromYaml -}} diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml index 7f3cec8ff..0fd02b8a8 100644 --- a/packages/system/seaweedfs/tests/lifecycle_test.yaml +++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml @@ -57,6 +57,36 @@ tests: path: spec.template.spec.containers[0].command[2] pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 ' + # Both are the weed binary under a fixed memory limit the parent chart does + # not set, so their GOMEMLIMIT is pinned here rather than derived: without + # one the heap has no ceiling and a large lifecycle pass grows it past the + # limit. + - it: caps the admin heap below its memory limit + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 256Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 192MiB + + - it: caps the worker heap below its memory limit + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 384MiB + # The worker executes every job type it registers for. Lifecycle is the only # one this release asks for: vacuum, balance and erasure coding move data and # are not something enabling lifecycle should start doing as a side effect. diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 64e2562ba..81b486002 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -349,6 +349,11 @@ seaweedfs: limits: cpu: "500m" memory: "256Mi" + # 75% of the limit above, as cozy-lib.resources.goMemLimit derives it for + # the components whose resources the parent chart sets. Without it the + # heap has no absolute ceiling and grows past the limit. + extraEnvironmentVars: + GOMEMLIMIT: "192MiB" worker: enabled: false # Vacuum, balance and erasure coding move data; enabling lifecycle must @@ -378,6 +383,9 @@ seaweedfs: limits: cpu: "1" memory: "512Mi" + # 75% of the limit above; see admin.extraEnvironmentVars. + extraEnvironmentVars: + GOMEMLIMIT: "384MiB" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml new file mode 100644 index 000000000..f9d3739b0 --- /dev/null +++ b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-gomemlimit +data: +{{- range $name, $resources := .Values.sanitizedResources }} + {{ $name }}: {{ include "cozy-lib.resources.goMemLimit" $resources | quote }} +{{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml new file mode 100644 index 000000000..1753f925f --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml @@ -0,0 +1,62 @@ +# ./tests/resources_gomemlimit_test.yaml +suite: resources goMemLimit helper + +# The helper turns a memory limit into a GOMEMLIMIT the Go runtime accepts, +# at 75% of the limit. Two properties matter and neither is visible from a +# single value: the result carries the MiB suffix Go parses, and it stays +# strictly under the limit it protects, whatever unit the quantity came in. +# A value that came out at or above the limit would leave the runtime with +# no headroom and the container would still be killed. + +templates: + - templates/tests/resources-gomemlimit.yaml + +tests: + - it: takes 75% of the limit and keeps the unit Go parses + values: + - resources_gomemlimit_values.yaml + + release: + name: myrelease + namespace: default + + asserts: + - equal: + path: data.binaryGi + value: "6144MiB" + + - equal: + path: data.binaryMi + value: "384MiB" + + # A decimal quantity is smaller than its binary lookalike: 1G is + # 1000000000 bytes, so 75% of it lands below 768MiB. + - equal: + path: data.decimalG + value: "715MiB" + + - equal: + path: data.bareBytes + value: "768MiB" + + # The smallest limit that still yields a usable value: 75% of + # 1398102 bytes (4/3 Mi, rounded up) floors to 1MiB. + - equal: + path: data.smallestUsable + value: "1MiB" + + # Integer division floors, so a limit one byte under 4/3 Mi comes out + # at 0MiB. Go accepts that and then collects almost without pause, + # slowing the component instead of protecting it, so the render stops + # instead. + - it: refuses a limit too small to leave a Go heap + values: + - resources_gomemlimit_values.yaml + set: + sanitizedResources: + belowOneMiB: + limits: + memory: "1398101" + asserts: + - failedTemplate: + errorPattern: "leaves no room for a Go heap" diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml new file mode 100644 index 000000000..f6be917e0 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml @@ -0,0 +1,16 @@ +sanitizedResources: + binaryGi: + limits: + memory: "8Gi" + binaryMi: + limits: + memory: "512Mi" + decimalG: + limits: + memory: "1G" + bareBytes: + limits: + memory: "1073741824" + smallestUsable: + limits: + memory: "1398102"