From 079ec7c1f5c067181f28caa18dfa7f1e403e6fca Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Sat, 18 Apr 2026 14:54:24 +0300 Subject: [PATCH] fix(kubernetes): add kubelet reserved resources and cgroup enforcement Configure kubelet with proper resource reservations and cgroup enforcement to prevent OOM kills when memory pressure occurs. Configuration: - systemReserved: 100m CPU, 512Mi memory (OS daemons) - kubeReserved: 100m CPU, 512Mi memory (kubelet, containerd) - evictionHard: 512Mi threshold (immediate eviction) - evictionSoft: 1Gi threshold with 90s grace period - evictionMinimumReclaim: 256Mi (guaranteed reclaim after eviction) - enforce-node-allocatable: creates hard cgroup limits for reservations - cgroups-per-qos: enables QoS-based resource isolation - cgroup-driver: systemd (matches containerd configuration) - kernel-memcg-notification: faster memory pressure detection Allocatable calculation for 16Gi VM: VM capacity: 16.00 Gi - systemReserved: -0.50 Gi (enforced via cgroup) - kubeReserved: -0.50 Gi (enforced via cgroup) - evictionHard buffer: -0.50 Gi - kernel/page cache: -0.25 Gi (implicit) ---------------------------------------- Allocatable (safe): 14.25 Gi Key improvement: enforce-node-allocatable creates hard cgroup limits, preventing system/kubelet processes from exceeding reservations and killing kubelet via OOM. Related issue: tenant VMs experiencing kubelet crashes after 15-30min due to insufficient reserved memory (100Mi vs required 1.75Gi total) and lack of cgroup enforcement. Signed-off-by: IvanHunters --- packages/apps/kubernetes/templates/cluster.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 323d8621b..12bb77aae 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -283,6 +283,13 @@ spec: system-reserved: "memory={{ $systemReservedMemory }},cpu={{ $systemReservedCpu }}" kube-reserved: "memory={{ $kubeReservedMemory }},cpu={{ $kubeReservedCpu }}" eviction-hard: "memory.available<{{ $evictionHardMemory }}" + eviction-soft: "memory.available<1Gi" + eviction-soft-grace-period: "memory.available=1m30s" + eviction-minimum-reclaim: "memory.available=256Mi" + enforce-node-allocatable: "pods,system-reserved,kube-reserved" + cgroups-per-qos: "true" + cgroup-driver: "systemd" + kernel-memcg-notification: "true" # Ignore this for 1.31 ignorePreflightErrors: - FileExisting-conntrack