From 55066b981d3a93aeb395a529506d765aae6278da Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Thu, 29 Oct 2026 10:06:40 +0100 Subject: [PATCH] fix(kubernetes): bring up a replacement worker before draining the old one A worker pool rollout (instance type, image or version change) ran with maxUnavailable 1, so CAPI drained and deleted an old Machine as soon as its replacement had been created rather than once it had joined. A one-node pool was left with no node for the few minutes the new VM took to boot and register, and every pool lost one node of capacity for each step of the rollout. With maxUnavailable 0 a node is only drained once its replacement is up. The surge stays at the pool ceiling, with a floor of 1 because CAPI refuses a rolling update where both values are 0 and maxReplicas may be 0. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../kubernetes-nodes/templates/nodegroup.yaml | 9 ++- .../render_snapshot_test.yaml.snap | 6 +- .../tests/rollout_strategy_test.yaml | 67 +++++++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 18ec0e16b..534b93170 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1024,10 +1024,15 @@ spec: # action until the autoscaler notices unschedulable Pods and # scales back up — a real disruption window on every routine # platform bump. + {{- /* maxUnavailable 0 makes a rollout (instance type, image, version) + wait for each replacement Machine to be up before draining an old + one; with 1, a one-node pool lost its only node until the new one + joined. CAPI refuses maxSurge and maxUnavailable both at 0, and + maxReplicas may be 0, hence the floor of 1. */}} strategy: rollingUpdate: - maxSurge: {{ $group.maxReplicas }} - maxUnavailable: 1 + maxSurge: {{ max 1 ($group.maxReplicas | int) }} + maxUnavailable: 0 type: RollingUpdate selector: matchLabels: diff --git a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap index 281fc1238..8c7183b22 100644 --- a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap +++ b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap @@ -97,7 +97,7 @@ matches the golden render of a GPU pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: @@ -250,7 +250,7 @@ matches the golden render of a kubelet-override pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: @@ -405,7 +405,7 @@ matches the golden render of the pool objects: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml new file mode 100644 index 000000000..1741d77e1 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -0,0 +1,67 @@ +suite: worker pool rollout keeps capacity +templates: + - templates/nodegroup.yaml +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 1 + instanceType: "" + diskSize: 20Gi + storageClass: replicated + resources: + cpu: "2" + memory: 4Gi +tests: + # A rollout replaces every Machine. With maxUnavailable 1 a one-node pool + # drained and deleted its only node as soon as the replacement Machine was + # created, leaving the pool empty until the new node joined. + - it: never drains a node before its replacement is ready + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + + - it: surges up to the pool ceiling + set: + maxReplicas: 5 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 5 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + # CAPI rejects a RollingUpdate with maxSurge and maxUnavailable both 0. + - it: keeps a surge of one on a pool capped at zero + set: + minReplicas: 0 + maxReplicas: 0 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0