diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 18ae6255f..8e8653a99 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -272,7 +272,7 @@ When `systemReservedMemory` or `kubeReservedMemory` are left empty, they are **a - If neither is available, the reservation falls back to the minimum (256Mi). 2. Calculate **5%** of the effective memory (in MiB, rounded down). 3. **Clamp** the result to the range **[256Mi, 1Gi]**: - - Nodes with 5.12 GiB or less get the minimum 256Mi reservation. + - Nodes with 5 GiB or less get the minimum 256Mi reservation. - Nodes with 20 GiB or more get the maximum 1Gi reservation. Both `systemReservedMemory` and `kubeReservedMemory` receive the same auto-computed value by default. @@ -294,6 +294,12 @@ Eviction thresholds can be specified as percentages (e.g., `7%`) or absolute val The `evictionSoftGracePeriod` and `evictionMinimumReclaim` parameters are currently hardcoded in the chart template and cannot be overridden through values. +#### Capacity Annotation + +When kubelet resource reservations are configured, the `capacity.cluster-autoscaler.kubernetes.io/memory` annotation on worker nodes reports **allocatable memory** (total node memory minus system-reserved, kube-reserved, and eviction-hard) instead of total node memory. This aligns the annotation with the value the cluster autoscaler uses in its scaling calculations. + +Upgrading from a version without this feature may cause the autoscaler to see reduced per-node capacity after the annotations are updated, which can trigger additional scale-up operations. No action is typically required — the new values reflect the actual memory available for workload scheduling. + #### Example: Override kubelet reservations ```yaml diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index ba410f473..a427de0c2 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -211,9 +211,10 @@ metadata: {{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} {{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }} {{/* Validate reservation field formats. - These guards validate the USER-SUPPLIED value (from values.yaml via - $kubeletOverride), not the resolved/computed default. The resolved - default is always valid by construction, so only user input needs validation. */}} + These guards check the USER-SUPPLIED value (from values.yaml via + $kubeletOverride), not the resolved/computed default — the default is + always valid by construction. The eviction threshold validation below + differs: it validates the RESOLVED values (user-supplied or default). */}} {{- $validations := list (dict "key" "systemReservedMemory" "value" $kubeletOverride.systemReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)") (dict "key" "kubeReservedMemory" "value" $kubeletOverride.kubeReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)") @@ -237,8 +238,28 @@ metadata: {{- end }} {{- end }} {{- if and (hasSuffix "%" $evictionHardMemory) (hasSuffix "%" $evictionSoftMemory) }} -{{- $hardPct := trimSuffix "%" $evictionHardMemory | float64 }} -{{- $softPct := trimSuffix "%" $evictionSoftMemory | float64 }} +{{- $hardPctStr := trimSuffix "%" $evictionHardMemory }} +{{- $softPctStr := trimSuffix "%" $evictionSoftMemory }} +{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $hardPctStr) }} +{{- fail (printf "nodeGroup %s: evictionHardMemory percentage value %q is not a valid number" $groupName $hardPctStr) }} +{{- end }} +{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $softPctStr) }} +{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage value %q is not a valid number" $groupName $softPctStr) }} +{{- end }} +{{- $hardPct := $hardPctStr | float64 }} +{{- $softPct := $softPctStr | float64 }} +{{- if le $hardPct 0.0 }} +{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be between 1 and 100, got %s%%" $groupName $hardPctStr) }} +{{- end }} +{{- if gt $hardPct 100.0 }} +{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be between 1 and 100, got %s%%" $groupName $hardPctStr) }} +{{- end }} +{{- if le $softPct 0.0 }} +{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be between 1 and 100, got %s%%" $groupName $softPctStr) }} +{{- end }} +{{- if gt $softPct 100.0 }} +{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be between 1 and 100, got %s%%" $groupName $softPctStr) }} +{{- end }} {{- if ge $hardPct $softPct }} {{- fail (printf "nodeGroup %s: evictionHardMemory (%s) must be strictly less than evictionSoftMemory (%s)" $groupName $evictionHardMemory $evictionSoftMemory) }} {{- end }} @@ -272,6 +293,13 @@ metadata: {{- fail (printf "nodeGroup %s: total reserved memory (systemReservedMemory=%s + kubeReservedMemory=%s + evictionHardMemory=%s) must be less than effective node memory (%s)" $groupName $systemReservedMemory $kubeReservedMemory $evictionHardMemory $effectiveMemory) }} {{- end }} {{- end }} +{{/* Guard: explicit kubelet overrides require effective memory for validation */}} +{{- if not $effectiveMemory }} +{{- $hasOverride := or $kubeletOverride.systemReservedMemory $kubeletOverride.kubeReservedMemory $kubeletOverride.evictionHardMemory $kubeletOverride.evictionSoftMemory }} +{{- if $hasOverride }} +{{- fail (printf "nodeGroup %s: cannot validate kubelet reservations without effective memory — set resources.memory or a valid instanceType" $groupName) }} +{{- end }} +{{- end }} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -354,7 +382,6 @@ spec: cgroups-per-qos: "true" cgroup-driver: "systemd" kernel-memcg-notification: "true" - # Ignore this for 1.31 ignorePreflightErrors: - FileExisting-conntrack discovery: @@ -393,12 +420,8 @@ metadata: {{- $kubeBytes := include "cozy-lib.resources.toFloat" $kubeReservedMemory | float64 }} {{- $totalReservedBytes := addf (addf $sysBytes $kubeBytes) $evictionHardBytes }} {{- $allocatableBytes := subf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) $totalReservedBytes }} - {{- $allocatableMi := divf $allocatableBytes 1048576.0 | int }} + {{- $allocatableMi := divf $allocatableBytes 1048576.0 | ceil }} capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $allocatableMi }}Mi" - {{- else if and $group.resources $group.resources.memory }} - capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $group.resources.memory }}" - {{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} - capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $instanceType.spec.memory.guest }}" {{- end }} {{- if and $group.resources $group.resources.cpu }} capacity.cluster-autoscaler.kubernetes.io/cpu: "{{ $group.resources.cpu }}" diff --git a/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml b/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml index f2bc8bdf0..f916e17a9 100644 --- a/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml +++ b/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml @@ -177,7 +177,8 @@ tests: ephemeralStorage: 20Gi roles: - worker - resources: {} + resources: + memory: "8Gi" kubelet: systemReservedMemory: "512Mi" kubeReservedMemory: "256Mi" @@ -222,7 +223,8 @@ tests: ephemeralStorage: 20Gi roles: - worker - resources: {} + resources: + memory: "4Gi" kubelet: systemReservedMemory: "256Mi" asserts: @@ -262,7 +264,8 @@ tests: ephemeralStorage: 20Gi roles: - worker - resources: {} + resources: + memory: "8Gi" kubelet: evictionHardMemory: "200Mi" evictionSoftMemory: "500Mi" @@ -860,7 +863,8 @@ tests: ephemeralStorage: 40Gi roles: - worker - resources: {} + resources: + memory: "8Gi" kubelet: systemReservedMemory: "512Mi" kubeReservedMemory: "256Mi" @@ -997,7 +1001,8 @@ tests: ephemeralStorage: 20Gi roles: - worker - resources: {} + resources: + memory: "8Gi" kubelet: evictionHardMemory: "500Mi" evictionSoftMemory: "1Gi" @@ -1011,6 +1016,145 @@ tests: value: "memory.available<1Gi" documentIndex: 4 + #################################################### + # Eviction percentage range validation # + #################################################### + + - it: rejects non-numeric evictionHardMemory percentage + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "abc%" + evictionSoftMemory: "10%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory percentage value \"abc\" is not a valid number" + + - it: rejects evictionHardMemory percentage over 100 + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "150%" + evictionSoftMemory: "200%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory percentage must be between 1 and 100, got 150%" + + - it: rejects evictionSoftMemory percentage over 100 + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "5%" + evictionSoftMemory: "150%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionSoftMemory percentage must be between 1 and 100, got 150%" + + - it: rejects evictionHardMemory percentage of zero + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "0%" + evictionSoftMemory: "10%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory percentage must be between 1 and 100, got 0%" + + - it: rejects negative evictionHardMemory percentage + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "-5%" + evictionSoftMemory: "10%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory percentage value \"-5\" is not a valid number" + #################################################### # Reserved memory exceeds node memory validation # #################################################### @@ -1107,6 +1251,36 @@ tests: value: "memory=512Mi,cpu=100m" documentIndex: 4 + ################################################################ + # Overrides without effective memory must be rejected # + ################################################################ + + - it: rejects kubelet overrides when no effective memory is available + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + systemReservedMemory: "1Gi" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: cannot validate kubelet reservations without effective memory — set resources.memory or a valid instanceType" + ############################################################# # Eviction thresholds interaction with explicit reservations ############################################################# @@ -1202,9 +1376,196 @@ tests: documentIndex: 4 #################################################### - # instanceType validation: must exist in cluster # + # Capacity annotation: default auto-computation # #################################################### + - it: sets capacity.memory annotation with default auto-computed reservations (4Gi) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "4Gi" + asserts: + # 4Gi = 4096Mi; 5% = 204Mi -> clamped to 256Mi floor + # systemReserved=256Mi, kubeReserved=256Mi, evictionHard=7% of 4Gi + # allocatable = ceil(4096 - 256 - 256 - 286.72) = ceil(3297.28) = 3298Mi + - isKind: + of: MachineDeployment + documentIndex: 6 + - equal: + path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"] + value: "3298Mi" + documentIndex: 6 + + - it: sets capacity.memory annotation with explicit reservation overrides (8Gi) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "8Gi" + kubelet: + systemReservedMemory: "1Gi" + kubeReservedMemory: "512Mi" + asserts: + # 8Gi = 8192Mi; systemReserved=1Gi, kubeReserved=512Mi, evictionHard=7% of 8Gi + # allocatable = ceil(8192 - 1024 - 512 - 573.44) = ceil(6082.56) = 6083Mi + - isKind: + of: MachineDeployment + documentIndex: 6 + - equal: + path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"] + value: "6083Mi" + documentIndex: 6 + + - it: omits capacity.memory annotation when no memory info is available + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + asserts: + - isKind: + of: MachineDeployment + documentIndex: 6 + - isNull: + path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"] + documentIndex: 6 + + - it: omits capacity.cpu annotation when no cpu info is available + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + asserts: + - isKind: + of: MachineDeployment + documentIndex: 6 + - isNull: + path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/cpu"] + documentIndex: 6 + + #################################################### + # Capacity annotation: ceil prevents zero truncation + #################################################### + + - it: reports non-zero capacity annotation when reservations consume most memory + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "2.5Gi" + kubelet: + systemReservedMemory: "1Gi" + kubeReservedMemory: "1Gi" + asserts: + # 2.5Gi = 2560Mi. Reserved: 1Gi + 1Gi + 7% of 2560Mi (179.2Mi) = 2359.2Mi. + # Remaining: ~200.8Mi. ceil ensures at least 1Mi is reported instead of 0 from int truncation. + - isKind: + of: MachineDeployment + documentIndex: 6 + - notEqual: + path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"] + value: "0Mi" + documentIndex: 6 + - matchRegex: + path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"] + pattern: "^\\d+Mi$" + documentIndex: 6 + + #################################################### + # instanceType validation # + #################################################### + + # NOTE: The instanceType-based auto-computation path (where resources.memory is + # absent and instanceType points to a VirtualMachineClusterInstancetype CR whose + # .spec.memory.guest is used as $effectiveMemory) cannot be tested here. + # + # The template uses Helm's `lookup` function to fetch the instancetype CR + # (cluster.yaml line ~187). `lookup` requires a live Kubernetes API server and + # always returns empty in helm-unittest (v1.0.3 provides no lookup mocking). + # When lookup returns empty, the template calls `fail`, so the positive path + # (successful lookup -> reservation derived from spec.memory.guest) is + # unreachable in unit tests. + # + # All reservation auto-computation tests above use resources.memory directly, + # which exercises the same 5%-clamp logic that the instanceType path uses. + # The instanceType resolution itself is validated via e2e tests that deploy + # against a real cluster with instancetype CRs present. + - it: fails when instanceType is specified but not found in cluster release: name: test-k8s