diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 01cf736d6..0f9e6d572 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -4,6 +4,9 @@ KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml) include ../../../hack/common-envs.mk include ../../../hack/package.mk +test: + helm unittest . + generate: cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go ../../../hack/update-crd.sh diff --git a/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml b/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml new file mode 100644 index 000000000..fbad0984b --- /dev/null +++ b/packages/apps/kubernetes/tests/kubelet-reservation_test.yaml @@ -0,0 +1,695 @@ +suite: kubelet reservation tests + +templates: + - templates/cluster.yaml + +# The KubeadmConfigTemplate for the first nodeGroup (md0) is rendered +# at document index 4 within cluster.yaml. The kubelet args live at: +# spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs + +tests: + ################################### + # Default values (no memory set) # + ################################### + + - it: renders default kubelet args when no memory and no kubelet overrides + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + asserts: + - isKind: + of: KubeadmConfigTemplate + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard + value: "memory.available<7%" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft + value: "memory.available<10%" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft-grace-period + value: "memory.available=1m30s" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-minimum-reclaim + value: "memory.available=256Mi" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.enforce-node-allocatable + value: "pods" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.cgroups-per-qos + value: "true" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.cgroup-driver + value: "systemd" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kernel-memcg-notification + value: "true" + documentIndex: 4 + + ########################################## + # Empty kubelet: {} falls back to defaults + ########################################## + + - it: falls back to defaults when kubelet is empty dict + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: {} + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard + value: "memory.available<7%" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft + value: "memory.available<10%" + documentIndex: 4 + + ######################################### + # Node group without kubelet key at all # + ######################################### + + - it: renders defaults when nodeGroup has no kubelet key + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + + ############################### + # Custom kubelet overrides # + ############################### + + - it: uses custom kubelet overrides when specified + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + systemReservedMemory: "512Mi" + kubeReservedMemory: "256Mi" + systemReservedCpu: "200m" + kubeReservedCpu: "300m" + evictionHardMemory: "5%" + evictionSoftMemory: "15%" + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=512Mi,cpu=200m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=256Mi,cpu=300m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard + value: "memory.available<5%" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft + value: "memory.available<15%" + documentIndex: 4 + + - it: allows partial kubelet overrides, uses defaults for unset fields + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + systemReservedMemory: "256Mi" + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=256Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard + value: "memory.available<7%" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft + value: "memory.available<10%" + documentIndex: 4 + + - it: allows absolute eviction thresholds without percentage validation + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "500Mi" + evictionSoftMemory: "200Mi" + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-hard + value: "memory.available<500Mi" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft + value: "memory.available<200Mi" + documentIndex: 4 + + ############################################ + # Auto-computation from resources.memory # + ############################################ + + - it: auto-computes reservation from resources.memory (4Gi -> 204Mi) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "4Gi" + asserts: + # 4Gi = 4096Mi, 5% = 204Mi (truncated from 204.8) + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=204Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=204Mi,cpu=100m" + documentIndex: 4 + + - it: auto-computes reservation from resources.memory (8Gi -> 409Mi) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "8Gi" + asserts: + # 8Gi = 8192Mi, 5% = 409Mi (truncated from 409.6) + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=409Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=409Mi,cpu=100m" + documentIndex: 4 + + ################################## + # Clamp logic: 64Mi floor # + ################################## + + - it: clamps auto-computed reservation to 64Mi floor (1Gi memory) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "1Gi" + asserts: + # 1Gi = 1024Mi, 5% = 51Mi, but floor is 64Mi + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + + - it: clamps auto-computed reservation to 64Mi floor (512Mi memory) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "512Mi" + asserts: + # 512Mi, 5% = 25Mi, but floor is 64Mi + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=64Mi,cpu=100m" + documentIndex: 4 + + ################################## + # Clamp logic: 1024Mi ceiling # + ################################## + + - it: clamps auto-computed reservation to 1024Mi ceiling (32Gi memory) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "32Gi" + asserts: + # 32Gi = 32768Mi, 5% = 1638Mi, capped to 1024Mi + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=1024Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=1024Mi,cpu=100m" + documentIndex: 4 + + - it: clamps auto-computed reservation to 1024Mi ceiling (64Gi memory) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "64Gi" + asserts: + # 64Gi = 65536Mi, 5% = 3276Mi, capped to 1024Mi + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=1024Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=1024Mi,cpu=100m" + documentIndex: 4 + + ################################################## + # Boundary: exactly at 1024Mi ceiling threshold # + ################################################## + + - it: auto-computes exactly 1024Mi at the boundary (20480Mi memory) + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + # 20480Mi = 20Gi, 5% = 1024Mi — exactly at the cap + memory: "20Gi" + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=1024Mi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=1024Mi,cpu=100m" + documentIndex: 4 + + #################################################### + # Custom overrides take precedence over auto-compute + #################################################### + + - it: custom kubelet memory overrides take precedence over auto-computation + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "8Gi" + kubelet: + systemReservedMemory: "1Gi" + kubeReservedMemory: "768Mi" + asserts: + # Despite 8Gi yielding 409Mi auto, explicit overrides win + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.system-reserved + value: "memory=1Gi,cpu=100m" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kube-reserved + value: "memory=768Mi,cpu=100m" + documentIndex: 4 + + ###################################### + # Eviction validation: hard >= soft # + ###################################### + + - it: fails when evictionHardMemory percentage exceeds evictionSoftMemory + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "15%" + evictionSoftMemory: "10%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory (15%) must be strictly less than evictionSoftMemory (10%)" + + - it: fails when evictionHardMemory percentage equals evictionSoftMemory + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "10%" + evictionSoftMemory: "10%" + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory (10%) must be strictly less than evictionSoftMemory (10%)" + + - it: fails when default evictionHardMemory is overridden to exceed default evictionSoftMemory + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: {} + kubelet: + evictionHardMemory: "12%" + # soft defaults to 10%, so 12% >= 10% should fail + asserts: + - failedTemplate: + errorMessage: "nodeGroup md0: evictionHardMemory (12%) must be strictly less than evictionSoftMemory (10%)" + + ############################################## + # Static args always present # + ############################################## + + - it: always renders enforce-node-allocatable as pods + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + ephemeralStorage: 20Gi + roles: + - worker + resources: + memory: "16Gi" + kubelet: + systemReservedMemory: "1Gi" + kubeReservedMemory: "512Mi" + asserts: + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.enforce-node-allocatable + value: "pods" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.cgroups-per-qos + value: "true" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.cgroup-driver + value: "systemd" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.kernel-memcg-notification + value: "true" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-soft-grace-period + value: "memory.available=1m30s" + documentIndex: 4 + - equal: + path: spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.eviction-minimum-reclaim + value: "memory.available=256Mi" + documentIndex: 4