mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
The Talos userspace OOM handler (v1.12+) ranks cgroups with
memory_max.hasValue() ? 0.0 :
{Besteffort: 1.0, Burstable: 0.5, ...}[class] * memory_current
and discards every cgroup scoring zero, so only a memory limit takes a
pod out of the victim set. Selection is decoupled from the trigger: a
tenant workload thrashing against its own limit drives node memory PSI,
and a limit-free system DaemonSet is killed for it. metallb-speaker and
linstor-satellite were the observed victims.
Set requests and limits on the node DaemonSets that had none. Requests
come from 14d peak working set measured on a 3-node dev cluster; limits
are deliberately loose, sized as a runaway backstop rather than a fitted
ceiling, because several of these scale with cluster size that a dev
cluster understates and because undersizing converts a rare
pressure-driven kill into a deterministic one. A limit costs nothing
until it is hit -- only the request reserves.
memory.max lands on the POD cgroup only when EVERY container in the pod
carries a limit, so sidecars and init containers are covered too; one
limit-free container returns the whole pod to the victim set.
Two DaemonSets are not chart-rendered and needed their own seam:
linstor-satellite via the LinstorSatelliteConfiguration podTemplate, and
virt-handler via KubeVirt customizeComponents.patches, since virt-operator
owns that DaemonSet. Note linstor-controller stays deliberately limit-free
-- it is a JVM that must not be OOM-killed mid-operation, and cluster_test
pins that.
Two things the multus manifest needed beyond the values change.
customize-deployment.patch is the source for that template, so the
limits are carried there rather than only in the generated file, which
`make update` would discard. And the reason kube-multus had no memory
limit was that this patch DELETED upstream's 50Mi rather than raising
it -- 50Mi being well under the 230Mi measured peak -- which is what
removed memory.max from the pod cgroup in the first place.
Not covered, because upstream hardcodes the containers with no resources
knob: the four frr-k8s cp-* init containers, the kube-ovn hostpath-init
and install-cni init containers, and cozy-proxy, whose vendored chart
exposes no resources value at all. hami and kilo are left alone as
optional components with no usage measurements behind them. All of these
still get a limit from the default LimitRange the operator maintains in
every system namespace.
Every limit is pinned by a helm-unittest assertion, and every assertion
was mutation-tested: changing the rendered value turns its suite red.
Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
177 lines
7.2 KiB
Diff
177 lines
7.2 KiB
Diff
--- a/packages/system/multus/templates/multus-daemonset-thick.yml
|
|
+++ b/packages/system/multus/templates/multus-daemonset-thick.yml
|
|
@@ -94,19 +94,19 @@
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: multus
|
|
- namespace: kube-system
|
|
+ namespace: cozy-multus
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: multus
|
|
- namespace: kube-system
|
|
+ namespace: cozy-multus
|
|
---
|
|
kind: ConfigMap
|
|
apiVersion: v1
|
|
metadata:
|
|
name: multus-daemon-config
|
|
- namespace: kube-system
|
|
+ namespace: cozy-multus
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
@@ -120,14 +120,15 @@
|
|
"cniConfigDir": "/host/etc/cni/net.d",
|
|
"multusAutoconfigDir": "/host/etc/cni/net.d",
|
|
"multusConfigFile": "auto",
|
|
+ "multusMasterCNI": "05-cilium.conflist",
|
|
"socketDir": "/host/run/multus/"
|
|
}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
- name: kube-multus-ds
|
|
- namespace: kube-system
|
|
+ name: cozy-multus
|
|
+ namespace: cozy-multus
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
@@ -157,13 +158,33 @@
|
|
- name: kube-multus
|
|
image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
|
|
command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
|
|
+ # Upstream sets 50Mi for both the request and the limit. This patch
|
|
+ # already raised the request to 100Mi and DELETED the memory limit
|
|
+ # outright, leaving only the CPU one -- which left the pod cgroup with
|
|
+ # no memory.max, and the Talos userspace OOM handler (v1.12+) only
|
|
+ # ever kills cgroups that lack it. So dropping the limit to survive
|
|
+ # 50Mi is what made multus an eviction candidate on every node.
|
|
+ #
|
|
+ # Restore a limit, sized as a backstop rather than a ceiling: 14d peak
|
|
+ # is 230Mi on a 3-node dev cluster and this grows with pods per node,
|
|
+ # so upstream's 50Mi was never survivable and a fitted value would
|
|
+ # trade a rare pressure-driven kill for a deterministic one.
|
|
+ #
|
|
+ # 1Gi rather than a tighter multiple of that peak, because the peak is
|
|
+ # the part least worth trusting here: multus-daemon holds per-pod
|
|
+ # state and a 3-node dev cluster understates pods per node more than
|
|
+ # it understates anything else measured for this change. The scar this
|
|
+ # avoids is the vertical-pod-autoscaler updater, which OOM-looped at a
|
|
+ # 110Mi limit for the same reason -- it scaled with pods in the
|
|
+ # cluster -- and was given a limit 2.5x its measured request to stop
|
|
+ # it. A limit costs nothing until it is hit; only the request reserves.
|
|
resources:
|
|
requests:
|
|
cpu: "100m"
|
|
- memory: "50Mi"
|
|
+ memory: "100Mi"
|
|
limits:
|
|
cpu: "100m"
|
|
- memory: "50Mi"
|
|
+ memory: "1Gi"
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
@@ -209,10 +230,14 @@
|
|
- "/host/opt/cni/bin"
|
|
- "-t"
|
|
- "thick"
|
|
+ # Short-lived, but it still needs a limit for the pod cgroup to get
|
|
+ # memory.max — the init container counts.
|
|
resources:
|
|
requests:
|
|
cpu: "10m"
|
|
memory: "15Mi"
|
|
+ limits:
|
|
+ memory: "128Mi"
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
@@ -220,6 +245,81 @@
|
|
- name: cnibin
|
|
mountPath: /host/opt/cni/bin
|
|
mountPropagation: Bidirectional
|
|
+{{- $stageCniPlugins := lower (ternary "false" (printf "%v" .Values.stageCniPlugins) (kindIs "invalid" .Values.stageCniPlugins)) }}
|
|
+{{- if not (has $stageCniPlugins (list "true" "yes" "on" "1" "false" "no" "off" "0")) }}{{- fail (printf "stageCniPlugins: expected a boolean, got %q" $stageCniPlugins) }}{{- end }}
|
|
+{{- if has $stageCniPlugins (list "true" "yes" "on" "1") }}
|
|
+ - name: install-cni-plugins
|
|
+ image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
|
|
+ command:
|
|
+ - "sh"
|
|
+ - "-c"
|
|
+ - |
|
|
+ set -eu
|
|
+ # cilium and kube-ovn write into this directory too, and a
|
|
+ # plugin here may be executing, so each install is a rename.
|
|
+ trap 'rm -rf /host/opt/cni/bin/.tmp-cozystack-multus-* || :' EXIT
|
|
+ if [ ! -d /cni-plugins ]; then
|
|
+ echo "no /cni-plugins in this image; skipping staging" >&2
|
|
+ exit 0
|
|
+ fi
|
|
+ failed=
|
|
+ for plugin in /cni-plugins/*; do
|
|
+ [ -e "$plugin" ] || continue
|
|
+ name="${plugin##*/}"
|
|
+ tmp="/host/opt/cni/bin/.tmp-cozystack-multus-${name}"
|
|
+ # A directory at either end makes every step succeed while
|
|
+ # installing nothing.
|
|
+ rm -rf "$tmp" || { failed="${failed} ${name}"; continue; }
|
|
+ if [ -d "/host/opt/cni/bin/${name}" ]; then
|
|
+ echo "${name} is a directory, not a plugin binary; left as is" >&2
|
|
+ failed="${failed} ${name}"
|
|
+ continue
|
|
+ fi
|
|
+ if cp -f "$plugin" "$tmp" && chmod 0755 "$tmp" &&
|
|
+ mv -f "$tmp" "/host/opt/cni/bin/${name}"; then
|
|
+ continue
|
|
+ fi
|
|
+ rm -rf "$tmp" || :
|
|
+ failed="${failed} ${name}"
|
|
+ done
|
|
+ if [ -n "$failed" ]; then
|
|
+ echo "failed to install:${failed}" >&2
|
|
+ echo "those were left as they were" >&2
|
|
+ echo "the cause for each is earlier in this log" >&2
|
|
+ echo "failed to install:${failed}" > /dev/termination-log 2>/dev/null || true
|
|
+ fi
|
|
+ exit 0
|
|
+ # Short-lived, but it still needs a limit for the pod cgroup to get
|
|
+ # memory.max — the init container counts. Same 128Mi as the sibling
|
|
+ # install-multus-binary above: both only copy binaries to the host.
|
|
+ resources:
|
|
+ requests:
|
|
+ cpu: "10m"
|
|
+ memory: "15Mi"
|
|
+ limits:
|
|
+ memory: "128Mi"
|
|
+ # cilium's install-cni-binaries copies plugin binaries into this same
|
|
+ # directory with ALL capabilities dropped, in the spc_t domain, with
|
|
+ # no privileged flag. On a node with SELinux enforcing, a container
|
|
+ # that has dropped every capability still cannot write a host-labelled
|
|
+ # directory without a domain that permits it.
|
|
+ #
|
|
+ # mountPropagation is absent: it governs mounts made INSIDE the
|
|
+ # container, which this script does not make, and the API server
|
|
+ # rejects Bidirectional unless the container is privileged.
|
|
+ securityContext:
|
|
+ allowPrivilegeEscalation: false
|
|
+ capabilities:
|
|
+ drop:
|
|
+ - ALL
|
|
+ seLinuxOptions:
|
|
+ level: s0
|
|
+ type: spc_t
|
|
+ terminationMessagePolicy: FallbackToLogsOnError
|
|
+ volumeMounts:
|
|
+ - name: cnibin
|
|
+ mountPath: /host/opt/cni/bin
|
|
+{{- end }}
|
|
terminationGracePeriodSeconds: 10
|
|
volumes:
|
|
- name: cni
|