--- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -94,19 +94,19 @@ subjects: - kind: ServiceAccount name: multus - namespace: kube-system + namespace: cozy-multus --- apiVersion: v1 kind: ServiceAccount metadata: name: multus - namespace: kube-system + namespace: cozy-multus --- kind: ConfigMap apiVersion: v1 metadata: name: multus-daemon-config - namespace: kube-system + namespace: cozy-multus labels: tier: node app: multus @@ -120,14 +120,15 @@ "cniConfigDir": "/host/etc/cni/net.d", "multusAutoconfigDir": "/host/etc/cni/net.d", "multusConfigFile": "auto", + "multusMasterCNI": "05-cilium.conflist", "socketDir": "/host/run/multus/" } --- apiVersion: apps/v1 kind: DaemonSet metadata: - name: kube-multus-ds - namespace: kube-system + name: cozy-multus + namespace: cozy-multus labels: tier: node app: multus @@ -157,13 +158,32 @@ - name: kube-multus image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick command: [ "/usr/src/multus-cni/bin/multus-daemon" ] + # Keep a memory limit: without one the pod cgroup has no memory.max, + # and the Talos userspace OOM handler (v1.12+) only ever kills cgroups + # that lack it, which makes multus an eviction candidate on every node. + # + # Size it as a backstop rather than a ceiling: 14d peak + # is 230Mi on a 3-node dev cluster and this grows with pods per node, + # so upstream's 50Mi was never survivable and a fitted value would + # trade a rare pressure-driven kill for a deterministic one. + # + # Far above that peak rather than a tighter multiple of it, because the + # peak is the part least worth trusting here: multus-daemon holds + # per-pod state and a 3-node dev cluster understates pods per node more + # than it understates anything else measured here. The scar + # this avoids is the vertical-pod-autoscaler updater, which OOM-looped + # at a 110Mi limit for the same reason -- it scaled with pods in the + # cluster -- and was given a limit 2.5x its measured request to stop + # it. A limit costs nothing until it is hit; only the request reserves. + # + # The values themselves, and why there is no CPU limit, are in + # values.yaml. resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" + {{- if not (kindIs "map" .Values.resources) }}{{- fail (printf "resources: expected a map, got %s" (kindOf .Values.resources)) }}{{- end }} + {{- range $key := list "limits" "requests" }}{{- $value := index $.Values.resources $key }}{{- if not (or (kindIs "invalid" $value) (kindIs "map" $value)) }}{{- fail (printf "resources.%s: expected a map, got %s" $key (kindOf $value)) }}{{- end }}{{- end }} + {{- if not (dig "limits" "memory" "" .Values.resources) }}{{- fail "resources.limits.memory must be set: without it the pod cgroup has no memory.max and the Talos OOM handler may evict multus on any node" }}{{- end }} + {{- if not (dig "requests" "memory" "" .Values.resources) }}{{- fail "resources.requests.memory must be set: without it the request defaults to the limit and a node with less allocatable memory is left without multus" }}{{- end }} + {{- toYaml .Values.resources | nindent 12 }} securityContext: privileged: true terminationMessagePolicy: FallbackToLogsOnError @@ -209,10 +229,14 @@ - "/host/opt/cni/bin" - "-t" - "thick" + # Short-lived, but it still needs a limit for the pod cgroup to get + # memory.max — the init container counts. resources: requests: cpu: "10m" memory: "15Mi" + limits: + memory: "128Mi" securityContext: privileged: true terminationMessagePolicy: FallbackToLogsOnError @@ -220,6 +244,81 @@ - name: cnibin mountPath: /host/opt/cni/bin mountPropagation: Bidirectional +{{- $stageCniPlugins := lower (ternary "false" (printf "%v" .Values.stageCniPlugins) (kindIs "invalid" .Values.stageCniPlugins)) }} +{{- if not (has $stageCniPlugins (list "true" "yes" "on" "1" "false" "no" "off" "0")) }}{{- fail (printf "stageCniPlugins: expected a boolean, got %q" $stageCniPlugins) }}{{- end }} +{{- if has $stageCniPlugins (list "true" "yes" "on" "1") }} + - name: install-cni-plugins + image: ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick + command: + - "sh" + - "-c" + - | + set -eu + # cilium and kube-ovn write into this directory too, and a + # plugin here may be executing, so each install is a rename. + trap 'rm -rf /host/opt/cni/bin/.tmp-cozystack-multus-* || :' EXIT + if [ ! -d /cni-plugins ]; then + echo "no /cni-plugins in this image; skipping staging" >&2 + exit 0 + fi + failed= + for plugin in /cni-plugins/*; do + [ -e "$plugin" ] || continue + name="${plugin##*/}" + tmp="/host/opt/cni/bin/.tmp-cozystack-multus-${name}" + # A directory at either end makes every step succeed while + # installing nothing. + rm -rf "$tmp" || { failed="${failed} ${name}"; continue; } + if [ -d "/host/opt/cni/bin/${name}" ]; then + echo "${name} is a directory, not a plugin binary; left as is" >&2 + failed="${failed} ${name}" + continue + fi + if cp -f "$plugin" "$tmp" && chmod 0755 "$tmp" && + mv -f "$tmp" "/host/opt/cni/bin/${name}"; then + continue + fi + rm -rf "$tmp" || : + failed="${failed} ${name}" + done + if [ -n "$failed" ]; then + echo "failed to install:${failed}" >&2 + echo "those were left as they were" >&2 + echo "the cause for each is earlier in this log" >&2 + echo "failed to install:${failed}" > /dev/termination-log 2>/dev/null || true + fi + exit 0 + # Short-lived, but it still needs a limit for the pod cgroup to get + # memory.max — the init container counts. Same 128Mi as the sibling + # install-multus-binary above: both only copy binaries to the host. + resources: + requests: + cpu: "10m" + memory: "15Mi" + limits: + memory: "128Mi" + # cilium's install-cni-binaries copies plugin binaries into this same + # directory with ALL capabilities dropped, in the spc_t domain, with + # no privileged flag. On a node with SELinux enforcing, a container + # that has dropped every capability still cannot write a host-labelled + # directory without a domain that permits it. + # + # mountPropagation is absent: it governs mounts made INSIDE the + # container, which this script does not make, and the API server + # rejects Bidirectional unless the container is privileged. + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin +{{- end }} terminationGracePeriodSeconds: 10 volumes: - name: cni