From e3e20180752219d23aff6c1ac26b909b9085b965 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 14 Sep 2026 10:25:13 +0200 Subject: [PATCH 1/2] fix(seaweedfs, cozy-lib): cap the Go heap so components survive their memory limit Every SeaweedFS component is a Go binary running under a cgroup memory limit, and none of them set GOMEMLIMIT. The collector then works from GOGC alone, a target relative to the live heap with no absolute ceiling, so a burst of allocation grows the heap past the limit and the kernel kills the container. Seen on an S3 gateway holding 6.4 GiB of heap_sys against an 8 GiB limit while only 389 MiB was actually in use: ten kills in seven days, each one an interruption of the object store every tenant writes its backups to. Raising the limit does not help, since nothing bounds the growth; it only moves the ceiling the heap climbs to. Derive the value from the limit already computed for each component, at the same 75% the javaHeap helper uses for the JVM. The remaining quarter covers what GOMEMLIMIT does not account for, goroutine stacks and mmap'd regions among them. Volume pools and zones carry their own value: the subchart merges volume into every entry of volumes, so a pool would otherwise inherit the value sized for the default volume's limit, above its own. The lifecycle admin and worker run the same binary under limits the parent chart does not set, so their value is pinned next to those limits. A limit under 2Mi floors to 0MiB, which Go accepts and then collects almost continuously; the render fails instead. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../extra/seaweedfs/templates/seaweedfs.yaml | 40 ++++- .../seaweedfs/tests/gomemlimit_test.yaml | 162 ++++++++++++++++++ .../library/cozy-lib/templates/_resources.tpl | 20 +++ .../seaweedfs/tests/lifecycle_test.yaml | 30 ++++ packages/system/seaweedfs/values.yaml | 8 + .../templates/tests/resources-gomemlimit.yaml | 8 + .../tests/resources_gomemlimit_test.yaml | 61 +++++++ .../tests/resources_gomemlimit_values.yaml | 16 ++ 8 files changed, 338 insertions(+), 7 deletions(-) create mode 100644 packages/extra/seaweedfs/tests/gomemlimit_test.yaml create mode 100644 packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml create mode 100644 packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml create mode 100644 packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 92b40ef4d..d69a3e493 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -248,13 +248,19 @@ spec: {{- /* An explicit null is dropped before schema validation, so neither bound sees it, and the master exits on the empty flag it renders. */}} volumeSizeLimitMB: {{ required "master.volumeSizeLimitMB must be set" .Values.master.volumeSizeLimitMB }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | nindent 10 }} +{{- $masterResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | fromYaml }} + resources: {{- toYaml $masterResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $masterResources | quote }} volume: {{ if eq .Values.topology "MultiZone" }} enabled: false {{- end }} replicas: {{ .Values.volume.replicas }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | nindent 10 }} +{{- $volumeResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | fromYaml }} + resources: {{- toYaml $volumeResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -273,7 +279,13 @@ spec: {{- range $poolName, $pool := .Values.volume.pools }} {{ $poolName }}: replicas: {{ ternary $pool.replicas $.Values.volume.replicas (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -289,7 +301,9 @@ spec: {{- range $zoneName, $zone := .Values.volume.zones }} {{ $zoneName }}: replicas: {{ ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list $.Values.volume.resourcesPreset $.Values.volume.resources $) | nindent 12 }} + resources: {{- toYaml $volumeResources | nindent 12 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -314,7 +328,13 @@ spec: {{- range $poolName, $pool := (dig "pools" dict $zone) }} {{ $zoneName }}-{{ $poolName }}: replicas: {{ ternary $pool.replicas (ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas")) (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -345,7 +365,10 @@ spec: replicas: {{ .Values.filer.replicas }} s3: domainName: {{ .Values.host | default (printf "s3.%s" $host) }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | nindent 10 }} +{{- $filerResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | fromYaml }} + resources: {{- toYaml $filerResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $filerResources | quote }} s3: replicas: {{ .Values.s3.replicas }} ingress: @@ -381,7 +404,10 @@ spec: {{- if not $wildcardSecret }} secretName: {{ .Release.Name }}-s3-ingress-tls {{- end }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }} +{{- $s3Resources := include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | fromYaml }} + resources: {{- toYaml $s3Resources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $s3Resources | quote }} admin: enabled: {{ .Values.lifecycle.enabled }} secret: diff --git a/packages/extra/seaweedfs/tests/gomemlimit_test.yaml b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml new file mode 100644 index 000000000..2b298b10e --- /dev/null +++ b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml @@ -0,0 +1,162 @@ +suite: seaweedfs components carry a GOMEMLIMIT derived from their memory limit +templates: + - templates/seaweedfs.yaml + +# Every SeaweedFS component is a Go binary. Without GOMEMLIMIT the runtime +# collects on a relative target and keeps no absolute ceiling, so a burst of +# allocation grows the heap past the cgroup limit and the kernel kills the +# container. Each component sizes its own value from its own memory limit, +# so the assertions below track the limit rather than a fixed string. + +tests: + - it: derives GOMEMLIMIT from the default preset on every component + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + asserts: + - equal: + path: spec.values.seaweedfs.master.resources.limits.memory + value: 512Mi + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.filer.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: follows the memory limit when a component is resized + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 8Gi + volume: + resources: + memory: 2Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 8Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 6144MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: keeps GOMEMLIMIT below the cgroup limit it protects + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 1Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 1Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 768MiB + + # The subchart merges volume into every entry of volumes, so a pool that + # did not carry its own value would run on the default volume's, sized for + # a larger limit than its own. + - it: sizes a Simple pool from its own limit, not the default volume's + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: sizes zones and zone pools from their own limits in MultiZone + documentSelector: + path: kind + value: HelmRelease + set: + topology: MultiZone + replicationFactor: 1 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + resources: + memory: 2Gi + zones: + a: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.volumes.a-small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.a-small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 48b0718fd..9b621fe0d 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -202,6 +202,26 @@ {{- printf `-Xms%dm -Xmx%dm` $xmsMi $xmxMi }} {{- end }} +{{- /* + goMemLimit takes a sanitized .Values.resources and returns a GOMEMLIMIT + value set to 75% of the memory limit. Without it the Go runtime collects + on a relative target (GOGC) with no absolute ceiling, so a burst of + allocation grows the heap past the cgroup limit and the kernel kills the + container. The 25% headroom covers what GOMEMLIMIT does not account for: + goroutine stacks and any mmap'd region. Accepts only sanitized resource + maps. +*/}} +{{- define "cozy-lib.resources.goMemLimit" }} +{{- $memoryLimitInt := include "cozy-lib.resources.toFloat" .limits.memory | float64 | int64 }} +{{- /* 4194304 is 4Mi */}} +{{- $limitMiB := div (mul $memoryLimitInt 3) 4194304 }} +{{- /* Go accepts 0, and then collects almost without pause. */}} +{{- if lt $limitMiB 1 }} +{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs at least 2Mi." .limits.memory) }} +{{- end }} +{{- printf `%dMiB` $limitMiB }} +{{- end }} + {{- define "cozy-lib.resources.flatten" -}} {{- $out := dict -}} {{- $res := include "cozy-lib.resources.sanitize" . | fromYaml -}} diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml index 7f3cec8ff..0fd02b8a8 100644 --- a/packages/system/seaweedfs/tests/lifecycle_test.yaml +++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml @@ -57,6 +57,36 @@ tests: path: spec.template.spec.containers[0].command[2] pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 ' + # Both are the weed binary under a fixed memory limit the parent chart does + # not set, so their GOMEMLIMIT is pinned here rather than derived: without + # one the heap has no ceiling and a large lifecycle pass grows it past the + # limit. + - it: caps the admin heap below its memory limit + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 256Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 192MiB + + - it: caps the worker heap below its memory limit + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 384MiB + # The worker executes every job type it registers for. Lifecycle is the only # one this release asks for: vacuum, balance and erasure coding move data and # are not something enabling lifecycle should start doing as a side effect. diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 64e2562ba..81b486002 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -349,6 +349,11 @@ seaweedfs: limits: cpu: "500m" memory: "256Mi" + # 75% of the limit above, as cozy-lib.resources.goMemLimit derives it for + # the components whose resources the parent chart sets. Without it the + # heap has no absolute ceiling and grows past the limit. + extraEnvironmentVars: + GOMEMLIMIT: "192MiB" worker: enabled: false # Vacuum, balance and erasure coding move data; enabling lifecycle must @@ -378,6 +383,9 @@ seaweedfs: limits: cpu: "1" memory: "512Mi" + # 75% of the limit above; see admin.extraEnvironmentVars. + extraEnvironmentVars: + GOMEMLIMIT: "384MiB" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml new file mode 100644 index 000000000..f9d3739b0 --- /dev/null +++ b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-gomemlimit +data: +{{- range $name, $resources := .Values.sanitizedResources }} + {{ $name }}: {{ include "cozy-lib.resources.goMemLimit" $resources | quote }} +{{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml new file mode 100644 index 000000000..17a7d0b97 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml @@ -0,0 +1,61 @@ +# ./tests/resources_gomemlimit_test.yaml +suite: resources goMemLimit helper + +# The helper turns a memory limit into a GOMEMLIMIT the Go runtime accepts, +# at 75% of the limit. Two properties matter and neither is visible from a +# single value: the result carries the MiB suffix Go parses, and it stays +# strictly under the limit it protects, whatever unit the quantity came in. +# A value that came out at or above the limit would leave the runtime with +# no headroom and the container would still be killed. + +templates: + - templates/tests/resources-gomemlimit.yaml + +tests: + - it: takes 75% of the limit and keeps the unit Go parses + values: + - resources_gomemlimit_values.yaml + + release: + name: myrelease + namespace: default + + asserts: + - equal: + path: data.binaryGi + value: "6144MiB" + + - equal: + path: data.binaryMi + value: "384MiB" + + # A decimal quantity is smaller than its binary lookalike: 1G is + # 1000000000 bytes, so 75% of it lands below 768MiB. + - equal: + path: data.decimalG + value: "715MiB" + + - equal: + path: data.bareBytes + value: "768MiB" + + # The smallest limit that still yields a usable value: 75% of 2Mi + # floors to 1MiB. + - equal: + path: data.twoMiB + value: "1MiB" + + # Integer division floors, so a limit under 2Mi comes out at 0MiB. Go + # accepts that and then collects almost without pause, slowing the + # component instead of protecting it, so the render stops instead. + - it: refuses a limit too small to leave a Go heap + values: + - resources_gomemlimit_values.yaml + set: + sanitizedResources: + belowOneMiB: + limits: + memory: "1Mi" + asserts: + - failedTemplate: + errorPattern: "leaves no room for a Go heap" diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml new file mode 100644 index 000000000..3e367cc21 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml @@ -0,0 +1,16 @@ +sanitizedResources: + binaryGi: + limits: + memory: "8Gi" + binaryMi: + limits: + memory: "512Mi" + decimalG: + limits: + memory: "1G" + bareBytes: + limits: + memory: "1073741824" + twoMiB: + limits: + memory: "2Mi" From 3b893c626af984810b87acae19dd7377f5eb874f Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 11:43:13 +0100 Subject: [PATCH 2/2] fix(cozy-lib): state the real lower bound in the goMemLimit error The limit that floors to 0MiB is anything under 4/3 Mi, not under 2Mi: 1536Ki still yields 1MiB. The message named the wrong bound and the tests pinned 2Mi and 1Mi, neither of which sits on the edge. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../library/cozy-lib/templates/_resources.tpl | 2 +- .../tests/resources_gomemlimit_test.yaml | 15 ++++++++------- .../tests/resources_gomemlimit_values.yaml | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 9b621fe0d..5830e1400 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -217,7 +217,7 @@ {{- $limitMiB := div (mul $memoryLimitInt 3) 4194304 }} {{- /* Go accepts 0, and then collects almost without pause. */}} {{- if lt $limitMiB 1 }} -{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs at least 2Mi." .limits.memory) }} +{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs a limit of at least 1398102 bytes (4/3 Mi) to come out at 1MiB." .limits.memory) }} {{- end }} {{- printf `%dMiB` $limitMiB }} {{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml index 17a7d0b97..1753f925f 100644 --- a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml @@ -39,15 +39,16 @@ tests: path: data.bareBytes value: "768MiB" - # The smallest limit that still yields a usable value: 75% of 2Mi - # floors to 1MiB. + # The smallest limit that still yields a usable value: 75% of + # 1398102 bytes (4/3 Mi, rounded up) floors to 1MiB. - equal: - path: data.twoMiB + path: data.smallestUsable value: "1MiB" - # Integer division floors, so a limit under 2Mi comes out at 0MiB. Go - # accepts that and then collects almost without pause, slowing the - # component instead of protecting it, so the render stops instead. + # Integer division floors, so a limit one byte under 4/3 Mi comes out + # at 0MiB. Go accepts that and then collects almost without pause, + # slowing the component instead of protecting it, so the render stops + # instead. - it: refuses a limit too small to leave a Go heap values: - resources_gomemlimit_values.yaml @@ -55,7 +56,7 @@ tests: sanitizedResources: belowOneMiB: limits: - memory: "1Mi" + memory: "1398101" asserts: - failedTemplate: errorPattern: "leaves no room for a Go heap" diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml index 3e367cc21..f6be917e0 100644 --- a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml @@ -11,6 +11,6 @@ sanitizedResources: bareBytes: limits: memory: "1073741824" - twoMiB: + smallestUsable: limits: - memory: "2Mi" + memory: "1398102"