diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index a427de0c2..6f1c324d2 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -189,25 +189,38 @@ metadata: {{- fail (printf "nodeGroup %s: specified instanceType %q not found in cluster" $groupName $group.instanceType) }} {{- end }} {{- end }} -{{/* Compute kubelet reservations: explicit resources.memory wins over instanceType.spec.memory.guest. - Auto-computed system/kube reserved = 5% of effective memory, clamped to [256Mi, 1Gi]. */}} +{{/* Compute kubelet reservations: explicit resources.memory/cpu wins over instanceType spec. + Auto-computed system/kube reserved memory = 5% of effective memory, clamped to [256Mi, 1Gi]. + Auto-computed system/kube reserved CPU = 5% of effective CPU, clamped to [50m, 500m]. */}} {{- $effectiveMemory := "" }} {{- if and $group.resources $group.resources.memory }} {{- $effectiveMemory = $group.resources.memory | toString }} {{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} {{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} {{- end }} +{{- $effectiveCpu := "" }} +{{- if and $group.resources $group.resources.cpu }} +{{- $effectiveCpu = $group.resources.cpu | toString }} +{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} +{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} +{{- end }} {{- $autoReservedMi := 256 }} {{- if $effectiveMemory }} {{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} {{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} {{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} {{- end }} +{{- $autoReservedMillicores := 50 }} +{{- if $effectiveCpu }} +{{- $effectiveCpuMillis := mulf ($effectiveCpu | float64) 1000.0 | int }} +{{- $fivePercentMillis := mulf ($effectiveCpuMillis | float64) 0.05 | int }} +{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} +{{- end }} {{- $kubeletOverride := $group.kubelet | default dict }} {{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} {{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} -{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default "100m" }} -{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default "100m" }} +{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} +{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} {{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} {{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }} {{/* Validate reservation field formats. @@ -293,6 +306,28 @@ metadata: {{- fail (printf "nodeGroup %s: total reserved memory (systemReservedMemory=%s + kubeReservedMemory=%s + evictionHardMemory=%s) must be less than effective node memory (%s)" $groupName $systemReservedMemory $kubeReservedMemory $evictionHardMemory $effectiveMemory) }} {{- end }} {{- end }} +{{/* Validate that total reserved CPU does not exceed effective node CPU */}} +{{- if $effectiveCpu }} +{{- $sysCpuStr := $systemReservedCpu | toString }} +{{- $kubeCpuStr := $kubeReservedCpu | toString }} +{{- $sysCpuMillis := 0.0 }} +{{- $kubeCpuMillis := 0.0 }} +{{- if hasSuffix "m" $sysCpuStr }} +{{- $sysCpuMillis = trimSuffix "m" $sysCpuStr | float64 }} +{{- else }} +{{- $sysCpuMillis = mulf ($sysCpuStr | float64) 1000.0 }} +{{- end }} +{{- if hasSuffix "m" $kubeCpuStr }} +{{- $kubeCpuMillis = trimSuffix "m" $kubeCpuStr | float64 }} +{{- else }} +{{- $kubeCpuMillis = mulf ($kubeCpuStr | float64) 1000.0 }} +{{- end }} +{{- $totalCpuMillis := addf $sysCpuMillis $kubeCpuMillis }} +{{- $effectiveCpuMillis := mulf ($effectiveCpu | float64) 1000.0 }} +{{- if ge $totalCpuMillis $effectiveCpuMillis }} +{{- fail (printf "nodeGroup %s: total reserved CPU (systemReservedCpu=%s + kubeReservedCpu=%s) must be less than effective node CPU (%s cores)" $groupName $systemReservedCpu $kubeReservedCpu $effectiveCpu) }} +{{- end }} +{{- end }} {{/* Guard: explicit kubelet overrides require effective memory for validation */}} {{- if not $effectiveMemory }} {{- $hasOverride := or $kubeletOverride.systemReservedMemory $kubeletOverride.kubeReservedMemory $kubeletOverride.evictionHardMemory $kubeletOverride.evictionSoftMemory }} @@ -300,6 +335,13 @@ metadata: {{- fail (printf "nodeGroup %s: cannot validate kubelet reservations without effective memory — set resources.memory or a valid instanceType" $groupName) }} {{- end }} {{- end }} +{{/* Guard: explicit kubelet CPU overrides require effective CPU for validation */}} +{{- if not $effectiveCpu }} +{{- $hasCpuOverride := or $kubeletOverride.systemReservedCpu $kubeletOverride.kubeReservedCpu }} +{{- if $hasCpuOverride }} +{{- fail (printf "nodeGroup %s: cannot validate kubelet CPU reservations without effective CPU — set resources.cpu or a valid instanceType" $groupName) }} +{{- end }} +{{- end }} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate diff --git a/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml b/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml index f916e17a9..ac0dc4864 100644 --- a/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml +++ b/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml @@ -38,11 +38,11 @@ tests: documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard @@ -105,11 +105,11 @@ tests: asserts: - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard @@ -147,11 +147,11 @@ tests: asserts: - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 ############################### @@ -179,6 +179,7 @@ tests: - worker resources: memory: "8Gi" + cpu: 4 kubelet: systemReservedMemory: "512Mi" kubeReservedMemory: "256Mi" @@ -230,11 +231,11 @@ tests: asserts: - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard @@ -335,11 +336,11 @@ tests: # 4Gi = 4096Mi, 5% = 204Mi (truncated from 204.8), clamped up to 256Mi floor - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - it: auto-computes reservation from resources.memory (8Gi -> 409Mi) @@ -367,11 +368,11 @@ tests: # 8Gi = 8192Mi, 5% = 409Mi (truncated from 409.6) - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=409Mi,cpu=100m" + value: "memory=409Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=409Mi,cpu=100m" + value: "memory=409Mi,cpu=50m" documentIndex: 4 ################################## @@ -403,11 +404,11 @@ tests: # 1Gi = 1024Mi, 5% = 51Mi, but floor is 256Mi - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - it: clamps auto-computed reservation to 256Mi floor (512Mi memory, triggers validation error) @@ -465,11 +466,11 @@ tests: # 32Gi = 32768Mi, 5% = 1638Mi, capped to 1024Mi - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=1024Mi,cpu=100m" + value: "memory=1024Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=1024Mi,cpu=100m" + value: "memory=1024Mi,cpu=50m" documentIndex: 4 - it: clamps auto-computed reservation to 1024Mi ceiling (64Gi memory) @@ -497,11 +498,11 @@ tests: # 64Gi = 65536Mi, 5% = 3276Mi, capped to 1024Mi - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=1024Mi,cpu=100m" + value: "memory=1024Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=1024Mi,cpu=100m" + value: "memory=1024Mi,cpu=50m" documentIndex: 4 ################################################## @@ -533,11 +534,11 @@ tests: asserts: - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=1024Mi,cpu=100m" + value: "memory=1024Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=1024Mi,cpu=100m" + value: "memory=1024Mi,cpu=50m" documentIndex: 4 #################################################### @@ -572,11 +573,11 @@ tests: # Despite 8Gi yielding 409Mi auto, explicit overrides win - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=1Gi,cpu=100m" + value: "memory=1Gi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=768Mi,cpu=100m" + value: "memory=768Mi,cpu=50m" documentIndex: 4 ###################################### @@ -865,6 +866,7 @@ tests: - worker resources: memory: "8Gi" + cpu: 4 kubelet: systemReservedMemory: "512Mi" kubeReservedMemory: "256Mi" @@ -883,11 +885,11 @@ tests: documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=256Mi,cpu=100m" + value: "memory=256Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard @@ -1244,11 +1246,11 @@ tests: asserts: - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=512Mi,cpu=100m" + value: "memory=512Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=512Mi,cpu=100m" + value: "memory=512Mi,cpu=50m" documentIndex: 4 ################################################################ @@ -1315,11 +1317,11 @@ tests: # 8Gi node, 1Gi reserved, 3Gi hard eviction — total 4Gi < 8Gi, passes validation. - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=512Mi,cpu=100m" + value: "memory=512Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=512Mi,cpu=100m" + value: "memory=512Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard @@ -1360,11 +1362,11 @@ tests: # 8Gi node, 1Gi reserved, 512Mi hard / 1Gi soft — typical production config. - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved - value: "memory=512Mi,cpu=100m" + value: "memory=512Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved - value: "memory=512Mi,cpu=100m" + value: "memory=512Mi,cpu=50m" documentIndex: 4 - equal: path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard @@ -1589,3 +1591,212 @@ tests: asserts: - failedTemplate: errorMessage: "nodeGroup md0: specified instanceType \"u1.medium\" not found in cluster" + + #################################################### + # CPU auto-computation from resources.cpu # + #################################################### + + - it: auto-computes CPU reservation from resources.cpu (4 cores -> 200m each) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "8Gi" + cpu: 4 + asserts: + # 4 cores = 4000m, 5% = 200m + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=409Mi,cpu=200m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=409Mi,cpu=200m" + documentIndex: 4 + + ################################## + # CPU clamp logic: 50m floor # + ################################## + + - it: clamps auto-computed CPU reservation to 50m floor (1 core) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "4Gi" + cpu: 1 + asserts: + # 1 core = 1000m, 5% = 50m — exactly at floor + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=256Mi,cpu=50m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=256Mi,cpu=50m" + documentIndex: 4 + + ################################## + # CPU clamp logic: 500m ceiling # + ################################## + + - it: clamps auto-computed CPU reservation to 500m ceiling (20 cores) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "32Gi" + cpu: 20 + asserts: + # 20 cores = 20000m, 5% = 1000m, capped to 500m + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=1024Mi,cpu=500m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=1024Mi,cpu=500m" + documentIndex: 4 + + #################################################### + # CPU total validation: exceeds effective CPU # + #################################################### + + - it: rejects reserved CPU exceeding effective node CPU + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "8Gi" + cpu: 2 + kubelet: + systemReservedCpu: "1200m" + kubeReservedCpu: "1200m" + asserts: + # 2 cores = 2000m, 1200+1200 = 2400 > 2000 -> fails + - failedTemplate: + errorMessage: "nodeGroup md0: total reserved CPU (systemReservedCpu=1200m + kubeReservedCpu=1200m) must be less than effective node CPU (2 cores)" + + ################################################################ + # CPU override guard: CPU overrides without effective CPU # + ################################################################ + + - it: rejects kubelet CPU overrides when no effective CPU is available + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + systemReservedCpu: "200m" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: cannot validate kubelet CPU reservations without effective CPU — set resources.cpu or a valid instanceType" + + ######################################################## + # Default CPU: no resources/instanceType -> 50m # + ######################################################## + + - it: defaults to 50m CPU reservation when no resources or instanceType + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=256Mi,cpu=50m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=256Mi,cpu=50m" + documentIndex: 4