admin/cozystack
1
0 Fork 0
mirror of https://github.com/cozystack/cozystack.git synced 2026-11-09 22:13:18 +09:00
Code Issues Packages Projects Releases Wiki Activity

fix(kubernetes): harden kubelet reservation validation and add missing tests

Browse Source 
Add eviction percentage validation (non-numeric, out-of-range, zero, negative),
guard against overrides without effective memory, use ceil instead of int for
allocatable annotation, remove dead code in capacity annotation branches, fix
README threshold (5.12 GiB -> 5 GiB), and document capacity annotation semantics.

Signed-off-by: Arsolitt <arsolitt@gmail.com>
This commit is contained in:
Arsolitt committed 2026-04-23 17:02:54 +03:00
1 parent 4292401d66
commit c968247dd8
3 files changed
+408 -18
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/apps/kubernetes/README.md
+7 -1
View File
@@ -272,7 +272,7 @@ When `systemReservedMemory` or `kubeReservedMemory` are left empty, they are **a
- If neither is available, the reservation falls back to the minimum (256Mi).
2. Calculate **5%** of the effective memory (in MiB, rounded down).
3. **Clamp** the result to the range **[256Mi, 1Gi]**:
- Nodes with 5.12 GiB or less get the minimum 256Mi reservation.
- Nodes with 5 GiB or less get the minimum 256Mi reservation.
- Nodes with 20 GiB or more get the maximum 1Gi reservation.
Both `systemReservedMemory` and `kubeReservedMemory` receive the same auto-computed value by default.
@@ -294,6 +294,12 @@ Eviction thresholds can be specified as percentages (e.g., `7%`) or absolute val
The `evictionSoftGracePeriod` and `evictionMinimumReclaim` parameters are currently hardcoded in the chart template and cannot be overridden through values.
#### Capacity Annotation
When kubelet resource reservations are configured, the `capacity.cluster-autoscaler.kubernetes.io/memory` annotation on worker nodes reports **allocatable memory** (total node memory minus system-reserved, kube-reserved, and eviction-hard) instead of total node memory. This aligns the annotation with the value the cluster autoscaler uses in its scaling calculations.
Upgrading from a version without this feature may cause the autoscaler to see reduced per-node capacity after the annotations are updated, which can trigger additional scale-up operations. No action is typically required — the new values reflect the actual memory available for workload scheduling.
#### Example: Override kubelet reservations
```yaml
packages/apps/kubernetes/templates/cluster.yaml
+34 -11
View File
@@ -211,9 +211,10 @@ metadata:
{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }}
{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }}
{{/* Validate reservation field formats.
These guards validate the USER-SUPPLIED value (from values.yaml via
$kubeletOverride), not the resolved/computed default. The resolved
default is always valid by construction, so only user input needs validation. */}}
These guards check the USER-SUPPLIED value (from values.yaml via
$kubeletOverride), not the resolved/computed default — the default is
always valid by construction. The eviction threshold validation below
differs: it validates the RESOLVED values (user-supplied or default). */}}
{{- $validations := list
(dict "key" "systemReservedMemory" "value" $kubeletOverride.systemReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)")
(dict "key" "kubeReservedMemory" "value" $kubeletOverride.kubeReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)")
@@ -237,8 +238,28 @@ metadata:
{{- end }}
{{- end }}
{{- if and (hasSuffix "%" $evictionHardMemory) (hasSuffix "%" $evictionSoftMemory) }}
{{- $hardPct := trimSuffix "%" $evictionHardMemory | float64 }}
{{- $softPct := trimSuffix "%" $evictionSoftMemory | float64 }}
{{- $hardPctStr := trimSuffix "%" $evictionHardMemory }}
{{- $softPctStr := trimSuffix "%" $evictionSoftMemory }}
{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $hardPctStr) }}
{{- fail (printf "nodeGroup %s: evictionHardMemory percentage value %q is not a valid number" $groupName $hardPctStr) }}
{{- end }}
{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $softPctStr) }}
{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage value %q is not a valid number" $groupName $softPctStr) }}
{{- end }}
{{- $hardPct := $hardPctStr | float64 }}
{{- $softPct := $softPctStr | float64 }}
{{- if le $hardPct 0.0 }}
{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be between 1 and 100, got %s%%" $groupName $hardPctStr) }}
{{- end }}
{{- if gt $hardPct 100.0 }}
{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be between 1 and 100, got %s%%" $groupName $hardPctStr) }}
{{- end }}
{{- if le $softPct 0.0 }}
{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be between 1 and 100, got %s%%" $groupName $softPctStr) }}
{{- end }}
{{- if gt $softPct 100.0 }}
{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be between 1 and 100, got %s%%" $groupName $softPctStr) }}
{{- end }}
{{- if ge $hardPct $softPct }}
{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) must be strictly less than evictionSoftMemory (%s)" $groupName $evictionHardMemory $evictionSoftMemory) }}
{{- end }}
@@ -272,6 +293,13 @@ metadata:
{{- fail (printf "nodeGroup %s: total reserved memory (systemReservedMemory=%s + kubeReservedMemory=%s + evictionHardMemory=%s) must be less than effective node memory (%s)" $groupName $systemReservedMemory $kubeReservedMemory $evictionHardMemory $effectiveMemory) }}
{{- end }}
{{- end }}
{{/* Guard: explicit kubelet overrides require effective memory for validation */}}
{{- if not $effectiveMemory }}
{{- $hasOverride := or $kubeletOverride.systemReservedMemory $kubeletOverride.kubeReservedMemory $kubeletOverride.evictionHardMemory $kubeletOverride.evictionSoftMemory }}
{{- if $hasOverride }}
{{- fail (printf "nodeGroup %s: cannot validate kubelet reservations without effective memory — set resources.memory or a valid instanceType" $groupName) }}
{{- end }}
{{- end }}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
@@ -354,7 +382,6 @@ spec:
cgroups-per-qos: "true"
cgroup-driver: "systemd"
kernel-memcg-notification: "true"
# Ignore this for 1.31
ignorePreflightErrors:
- FileExisting-conntrack
discovery:
@@ -393,12 +420,8 @@ metadata:
{{- $kubeBytes := include "cozy-lib.resources.toFloat" $kubeReservedMemory | float64 }}
{{- $totalReservedBytes := addf (addf $sysBytes $kubeBytes) $evictionHardBytes }}
{{- $allocatableBytes := subf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) $totalReservedBytes }}
{{- $allocatableMi := divf $allocatableBytes 1048576.0 | int }}
{{- $allocatableMi := divf $allocatableBytes 1048576.0 | ceil }}
capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $allocatableMi }}Mi"
{{- else if and $group.resources $group.resources.memory }}
capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $group.resources.memory }}"
{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }}
capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $instanceType.spec.memory.guest }}"
{{- end }}
{{- if and $group.resources $group.resources.cpu }}
capacity.cluster-autoscaler.kubernetes.io/cpu: "{{ $group.resources.cpu }}"
packages/apps/kubernetes/tests/kubelet-reservation_test.yaml
+367 -6
View File
@@ -177,7 +177,8 @@ tests:
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
resources:
memory: "8Gi"
kubelet:
systemReservedMemory: "512Mi"
kubeReservedMemory: "256Mi"
@@ -222,7 +223,8 @@ tests:
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
resources:
memory: "4Gi"
kubelet:
systemReservedMemory: "256Mi"
asserts:
@@ -262,7 +264,8 @@ tests:
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
resources:
memory: "8Gi"
kubelet:
evictionHardMemory: "200Mi"
evictionSoftMemory: "500Mi"
@@ -860,7 +863,8 @@ tests:
ephemeralStorage: 40Gi
roles:
- worker
resources: {}
resources:
memory: "8Gi"
kubelet:
systemReservedMemory: "512Mi"
kubeReservedMemory: "256Mi"
@@ -997,7 +1001,8 @@ tests:
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
resources:
memory: "8Gi"
kubelet:
evictionHardMemory: "500Mi"
evictionSoftMemory: "1Gi"
@@ -1011,6 +1016,145 @@ tests:
value: "memory.available<1Gi"
documentIndex: 4
####################################################
# Eviction percentage range validation #
####################################################
- it: rejects non-numeric evictionHardMemory percentage
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
kubelet:
evictionHardMemory: "abc%"
evictionSoftMemory: "10%"
asserts:
- failedTemplate:
errorMessage: "nodeGroup md0: evictionHardMemory percentage value \"abc\" is not a valid number"
- it: rejects evictionHardMemory percentage over 100
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
kubelet:
evictionHardMemory: "150%"
evictionSoftMemory: "200%"
asserts:
- failedTemplate:
errorMessage: "nodeGroup md0: evictionHardMemory percentage must be between 1 and 100, got 150%"
- it: rejects evictionSoftMemory percentage over 100
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
kubelet:
evictionHardMemory: "5%"
evictionSoftMemory: "150%"
asserts:
- failedTemplate:
errorMessage: "nodeGroup md0: evictionSoftMemory percentage must be between 1 and 100, got 150%"
- it: rejects evictionHardMemory percentage of zero
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
kubelet:
evictionHardMemory: "0%"
evictionSoftMemory: "10%"
asserts:
- failedTemplate:
errorMessage: "nodeGroup md0: evictionHardMemory percentage must be between 1 and 100, got 0%"
- it: rejects negative evictionHardMemory percentage
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
kubelet:
evictionHardMemory: "-5%"
evictionSoftMemory: "10%"
asserts:
- failedTemplate:
errorMessage: "nodeGroup md0: evictionHardMemory percentage value \"-5\" is not a valid number"
####################################################
# Reserved memory exceeds node memory validation #
####################################################
@@ -1107,6 +1251,36 @@ tests:
value: "memory=512Mi,cpu=100m"
documentIndex: 4
################################################################
# Overrides without effective memory must be rejected #
################################################################
- it: rejects kubelet overrides when no effective memory is available
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
kubelet:
systemReservedMemory: "1Gi"
asserts:
- failedTemplate:
errorMessage: "nodeGroup md0: cannot validate kubelet reservations without effective memory — set resources.memory or a valid instanceType"
#############################################################
# Eviction thresholds interaction with explicit reservations
#############################################################
@@ -1202,9 +1376,196 @@ tests:
documentIndex: 4
####################################################
# instanceType validation: must exist in cluster #
# Capacity annotation: default auto-computation #
####################################################
- it: sets capacity.memory annotation with default auto-computed reservations (4Gi)
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources:
memory: "4Gi"
asserts:
# 4Gi = 4096Mi; 5% = 204Mi -> clamped to 256Mi floor
# systemReserved=256Mi, kubeReserved=256Mi, evictionHard=7% of 4Gi
# allocatable = ceil(4096 - 256 - 256 - 286.72) = ceil(3297.28) = 3298Mi
- isKind:
of: MachineDeployment
documentIndex: 6
- equal:
path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"]
value: "3298Mi"
documentIndex: 6
- it: sets capacity.memory annotation with explicit reservation overrides (8Gi)
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources:
memory: "8Gi"
kubelet:
systemReservedMemory: "1Gi"
kubeReservedMemory: "512Mi"
asserts:
# 8Gi = 8192Mi; systemReserved=1Gi, kubeReserved=512Mi, evictionHard=7% of 8Gi
# allocatable = ceil(8192 - 1024 - 512 - 573.44) = ceil(6082.56) = 6083Mi
- isKind:
of: MachineDeployment
documentIndex: 6
- equal:
path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"]
value: "6083Mi"
documentIndex: 6
- it: omits capacity.memory annotation when no memory info is available
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
asserts:
- isKind:
of: MachineDeployment
documentIndex: 6
- isNull:
path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"]
documentIndex: 6
- it: omits capacity.cpu annotation when no cpu info is available
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources: {}
asserts:
- isKind:
of: MachineDeployment
documentIndex: 6
- isNull:
path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/cpu"]
documentIndex: 6
####################################################
# Capacity annotation: ceil prevents zero truncation
####################################################
- it: reports non-zero capacity annotation when reservations consume most memory
release:
name: test-k8s
namespace: tenant-test
set:
_namespace:
etcd: etcd
ingress: nginx
host: example.com
_cluster:
cluster-domain: cozy.local
nodeGroups:
md0:
minReplicas: 0
maxReplicas: 10
instanceType: ""
ephemeralStorage: 20Gi
roles:
- worker
resources:
memory: "2.5Gi"
kubelet:
systemReservedMemory: "1Gi"
kubeReservedMemory: "1Gi"
asserts:
# 2.5Gi = 2560Mi. Reserved: 1Gi + 1Gi + 7% of 2560Mi (179.2Mi) = 2359.2Mi.
# Remaining: ~200.8Mi. ceil ensures at least 1Mi is reported instead of 0 from int truncation.
- isKind:
of: MachineDeployment
documentIndex: 6
- notEqual:
path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"]
value: "0Mi"
documentIndex: 6
- matchRegex:
path: metadata.annotations["capacity.cluster-autoscaler.kubernetes.io/memory"]
pattern: "^\\d+Mi$"
documentIndex: 6
####################################################
# instanceType validation #
####################################################
# NOTE: The instanceType-based auto-computation path (where resources.memory is
# absent and instanceType points to a VirtualMachineClusterInstancetype CR whose
# .spec.memory.guest is used as $effectiveMemory) cannot be tested here.
#
# The template uses Helm's `lookup` function to fetch the instancetype CR
# (cluster.yaml line ~187). `lookup` requires a live Kubernetes API server and
# always returns empty in helm-unittest (v1.0.3 provides no lookup mocking).
# When lookup returns empty, the template calls `fail`, so the positive path
# (successful lookup -> reservation derived from spec.memory.guest) is
# unreachable in unit tests.
#
# All reservation auto-computation tests above use resources.memory directly,
# which exercises the same 5%-clamp logic that the instanceType path uses.
# The instanceType resolution itself is validated via e2e tests that deploy
# against a real cluster with instancetype CRs present.
- it: fails when instanceType is specified but not found in cluster
release:
name: test-k8s