From 392bbb3c96e278c3056ab98f5887d74771173374 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Thu, 29 Oct 2026 10:45:21 +0100 Subject: [PATCH] fix(kubernetes): keep maxUnavailable 1 on GPU worker pools A replacement VM in a pool with passed-through GPUs may need the very GPU the old node holds. With maxUnavailable 0 it would stay Pending and the rollout would never finish, so those pools keep removing a node first. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../kubernetes-nodes/templates/nodegroup.yaml | 6 ++++-- .../__snapshot__/render_snapshot_test.yaml.snap | 2 +- .../tests/rollout_strategy_test.yaml | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 534b93170..45ef30cc6 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1027,12 +1027,14 @@ spec: {{- /* maxUnavailable 0 makes a rollout (instance type, image, version) wait for each replacement Machine to be up before draining an old one; with 1, a one-node pool lost its only node until the new one - joined. CAPI refuses maxSurge and maxUnavailable both at 0, and + joined. A pool with passed-through GPUs keeps 1: its replacement VM + may need the very GPU the old node holds, and would wait for it + forever. CAPI refuses maxSurge and maxUnavailable both at 0, and maxReplicas may be 0, hence the floor of 1. */}} strategy: rollingUpdate: maxSurge: {{ max 1 ($group.maxReplicas | int) }} - maxUnavailable: 0 + maxUnavailable: {{ ternary 1 0 (not (empty $group.gpus)) }} type: RollingUpdate selector: matchLabels: diff --git a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap index 8c7183b22..c8e990a9c 100644 --- a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap +++ b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap @@ -97,7 +97,7 @@ matches the golden render of a GPU pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 0 + maxUnavailable: 1 type: RollingUpdate template: metadata: diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml index 1741d77e1..ae44bb03d 100644 --- a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -65,3 +65,20 @@ tests: - equal: path: spec.strategy.rollingUpdate.maxUnavailable value: 0 + + # A replacement VM in a pool with passed-through GPUs may need the very GPU + # the old node holds; with maxUnavailable 0 it would wait for it forever. + - it: lets a GPU pool remove a node before its replacement is up + set: + gpus: + - name: nvidia.com/AD102GL_L40S + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1