mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-13 08:05:19 +09:00
Aleksei Sviridkin
dca2e4ff0b
kube-multus serves the CNI call for every pod sandbox on its node, and its 100m CPU limit was hardcoded in the patched upstream template. After a CNI outage a node had ~600 dead sandboxes and 220 pods waiting: multus was throttled in every CFS period, calls outlived kubelet's two-minute CRI deadline, each retry grew the queue, and the node never recovered. With the CPU limit lifted the 1Gi memory limit OOM-looped; 2 CPU and 4Gi cleared the backlog in about three minutes. Resources now come from values, defaulting to the same requests, no CPU limit and a 4Gi memory limit. The memory limit stays so the pod cgroup keeps memory.max and remains outside the Talos OOM handler's candidate set. Since an explicit null in a values override could drop either memory key, the render now fails without limits.memory or requests.memory. Without the request it defaults to the limit, and a node with less allocatable memory than that is left without multus. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
180 lines
8.0 KiB
Diff
180 lines
8.0 KiB
Diff
--- a/packages/system/multus/templates/multus-daemonset-thick.yml
|
|
+++ b/packages/system/multus/templates/multus-daemonset-thick.yml
|
|
@@ -94,19 +94,19 @@
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: multus
|
|
- namespace: kube-system
|
|
+ namespace: cozy-multus
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: multus
|
|
- namespace: kube-system
|
|
+ namespace: cozy-multus
|
|
---
|
|
kind: ConfigMap
|
|
apiVersion: v1
|
|
metadata:
|
|
name: multus-daemon-config
|
|
- namespace: kube-system
|
|
+ namespace: cozy-multus
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
@@ -120,14 +120,15 @@
|
|
"cniConfigDir": "/host/etc/cni/net.d",
|
|
"multusAutoconfigDir": "/host/etc/cni/net.d",
|
|
"multusConfigFile": "auto",
|
|
+ "multusMasterCNI": "05-cilium.conflist",
|
|
"socketDir": "/host/run/multus/"
|
|
}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
- name: kube-multus-ds
|
|
- namespace: kube-system
|
|
+ name: cozy-multus
|
|
+ namespace: cozy-multus
|
|
labels:
|
|
tier: node
|
|
app: multus
|
|
@@ -157,13 +158,32 @@
|
|
- name: kube-multus
|
|
image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
|
|
command: [ "/usr/src/multus-cni/bin/multus-daemon" ]
|
|
+ # Keep a memory limit: without one the pod cgroup has no memory.max,
|
|
+ # and the Talos userspace OOM handler (v1.12+) only ever kills cgroups
|
|
+ # that lack it, which makes multus an eviction candidate on every node.
|
|
+ #
|
|
+ # Size it as a backstop rather than a ceiling: 14d peak
|
|
+ # is 230Mi on a 3-node dev cluster and this grows with pods per node,
|
|
+ # so upstream's 50Mi was never survivable and a fitted value would
|
|
+ # trade a rare pressure-driven kill for a deterministic one.
|
|
+ #
|
|
+ # Far above that peak rather than a tighter multiple of it, because the
|
|
+ # peak is the part least worth trusting here: multus-daemon holds
|
|
+ # per-pod state and a 3-node dev cluster understates pods per node more
|
|
+ # than it understates anything else measured here. The scar
|
|
+ # this avoids is the vertical-pod-autoscaler updater, which OOM-looped
|
|
+ # at a 110Mi limit for the same reason -- it scaled with pods in the
|
|
+ # cluster -- and was given a limit 2.5x its measured request to stop
|
|
+ # it. A limit costs nothing until it is hit; only the request reserves.
|
|
+ #
|
|
+ # The values themselves, and why there is no CPU limit, are in
|
|
+ # values.yaml.
|
|
resources:
|
|
- requests:
|
|
- cpu: "100m"
|
|
- memory: "50Mi"
|
|
- limits:
|
|
- cpu: "100m"
|
|
- memory: "50Mi"
|
|
+ {{- if not (kindIs "map" .Values.resources) }}{{- fail (printf "resources: expected a map, got %s" (kindOf .Values.resources)) }}{{- end }}
|
|
+ {{- range $key := list "limits" "requests" }}{{- $value := index $.Values.resources $key }}{{- if not (or (kindIs "invalid" $value) (kindIs "map" $value)) }}{{- fail (printf "resources.%s: expected a map, got %s" $key (kindOf $value)) }}{{- end }}{{- end }}
|
|
+ {{- if not (dig "limits" "memory" "" .Values.resources) }}{{- fail "resources.limits.memory must be set: without it the pod cgroup has no memory.max and the Talos OOM handler may evict multus on any node" }}{{- end }}
|
|
+ {{- if not (dig "requests" "memory" "" .Values.resources) }}{{- fail "resources.requests.memory must be set: without it the request defaults to the limit and a node with less allocatable memory is left without multus" }}{{- end }}
|
|
+ {{- toYaml .Values.resources | nindent 12 }}
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
@@ -209,10 +229,14 @@
|
|
- "/host/opt/cni/bin"
|
|
- "-t"
|
|
- "thick"
|
|
+ # Short-lived, but it still needs a limit for the pod cgroup to get
|
|
+ # memory.max — the init container counts.
|
|
resources:
|
|
requests:
|
|
cpu: "10m"
|
|
memory: "15Mi"
|
|
+ limits:
|
|
+ memory: "128Mi"
|
|
securityContext:
|
|
privileged: true
|
|
terminationMessagePolicy: FallbackToLogsOnError
|
|
@@ -220,6 +244,81 @@
|
|
- name: cnibin
|
|
mountPath: /host/opt/cni/bin
|
|
mountPropagation: Bidirectional
|
|
+{{- $stageCniPlugins := lower (ternary "false" (printf "%v" .Values.stageCniPlugins) (kindIs "invalid" .Values.stageCniPlugins)) }}
|
|
+{{- if not (has $stageCniPlugins (list "true" "yes" "on" "1" "false" "no" "off" "0")) }}{{- fail (printf "stageCniPlugins: expected a boolean, got %q" $stageCniPlugins) }}{{- end }}
|
|
+{{- if has $stageCniPlugins (list "true" "yes" "on" "1") }}
|
|
+ - name: install-cni-plugins
|
|
+ image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
|
|
+ command:
|
|
+ - "sh"
|
|
+ - "-c"
|
|
+ - |
|
|
+ set -eu
|
|
+ # cilium and kube-ovn write into this directory too, and a
|
|
+ # plugin here may be executing, so each install is a rename.
|
|
+ trap 'rm -rf /host/opt/cni/bin/.tmp-cozystack-multus-* || :' EXIT
|
|
+ if [ ! -d /cni-plugins ]; then
|
|
+ echo "no /cni-plugins in this image; skipping staging" >&2
|
|
+ exit 0
|
|
+ fi
|
|
+ failed=
|
|
+ for plugin in /cni-plugins/*; do
|
|
+ [ -e "$plugin" ] || continue
|
|
+ name="${plugin##*/}"
|
|
+ tmp="/host/opt/cni/bin/.tmp-cozystack-multus-${name}"
|
|
+ # A directory at either end makes every step succeed while
|
|
+ # installing nothing.
|
|
+ rm -rf "$tmp" || { failed="${failed} ${name}"; continue; }
|
|
+ if [ -d "/host/opt/cni/bin/${name}" ]; then
|
|
+ echo "${name} is a directory, not a plugin binary; left as is" >&2
|
|
+ failed="${failed} ${name}"
|
|
+ continue
|
|
+ fi
|
|
+ if cp -f "$plugin" "$tmp" && chmod 0755 "$tmp" &&
|
|
+ mv -f "$tmp" "/host/opt/cni/bin/${name}"; then
|
|
+ continue
|
|
+ fi
|
|
+ rm -rf "$tmp" || :
|
|
+ failed="${failed} ${name}"
|
|
+ done
|
|
+ if [ -n "$failed" ]; then
|
|
+ echo "failed to install:${failed}" >&2
|
|
+ echo "those were left as they were" >&2
|
|
+ echo "the cause for each is earlier in this log" >&2
|
|
+ echo "failed to install:${failed}" > /dev/termination-log 2>/dev/null || true
|
|
+ fi
|
|
+ exit 0
|
|
+ # Short-lived, but it still needs a limit for the pod cgroup to get
|
|
+ # memory.max — the init container counts. Same 128Mi as the sibling
|
|
+ # install-multus-binary above: both only copy binaries to the host.
|
|
+ resources:
|
|
+ requests:
|
|
+ cpu: "10m"
|
|
+ memory: "15Mi"
|
|
+ limits:
|
|
+ memory: "128Mi"
|
|
+ # cilium's install-cni-binaries copies plugin binaries into this same
|
|
+ # directory with ALL capabilities dropped, in the spc_t domain, with
|
|
+ # no privileged flag. On a node with SELinux enforcing, a container
|
|
+ # that has dropped every capability still cannot write a host-labelled
|
|
+ # directory without a domain that permits it.
|
|
+ #
|
|
+ # mountPropagation is absent: it governs mounts made INSIDE the
|
|
+ # container, which this script does not make, and the API server
|
|
+ # rejects Bidirectional unless the container is privileged.
|
|
+ securityContext:
|
|
+ allowPrivilegeEscalation: false
|
|
+ capabilities:
|
|
+ drop:
|
|
+ - ALL
|
|
+ seLinuxOptions:
|
|
+ level: s0
|
|
+ type: spc_t
|
|
+ terminationMessagePolicy: FallbackToLogsOnError
|
|
+ volumeMounts:
|
|
+ - name: cnibin
|
|
+ mountPath: /host/opt/cni/bin
|
|
+{{- end }}
|
|
terminationGracePeriodSeconds: 10
|
|
volumes:
|
|
- name: cni
|