From 5cd2502d01f011c718562c0fb12f3d680e5c76be Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 12 Aug 2026 11:00:34 +0300 Subject: [PATCH] feat(kubernetes): guard against pruning un-adopted worker pools on upgrade The Phase 2 split makes the parent chart render control-plane-only and relies on the platform migration (a pre-upgrade hook) having adopted each worker pool onto its child kubernetes-nodes release first. If that migration does not run (its version slot was already stamped, or it was renumbered out on a merge/backport), the live worker MachineDeployment is still owned by this release and Helm's control-plane-only re-render would prune it, cascade-deleting the tenant's worker VMs. Add a render-time guard: on an upgrade, when the render can see the cluster (release-namespace canary lookup non-nil, so client-side renders stay a no-op), look up worker MachineDeployments whose spec.clusterName is this cluster and whose meta.helm.sh/release-name is still this parent release (un-adopted) and fail with a legible message instead of rendering. The upgrade blocks with a reason and the workers keep running on the prior release until the migration runs. Discriminates on the adoption annotation, not mere existence, so it does not deadlock the routine upgrade (the MD always exists at render time); mirrors the fail-closed canary pattern of system/seaweedfs naming-guard. Inert offline; the live-fail path is covered by the upgrade-adoption e2e. Signed-off-by: IvanHunters --- .../apps/kubernetes/templates/cluster.yaml | 32 +++++++++++++++++++ .../tests/worker_adoption_guard_test.yaml | 30 +++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 436eab7d1..bfd79274a 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -77,6 +77,38 @@ data: to reconcile - this HelmRelease will pick up the DataStore on its next 5m reconcile loop and provision the cluster. {{- else }} +{{- /* Phase 2 worker-pool adoption guard. Worker pools now live in per-pool + kubernetes-nodes releases; the platform migration adopts each pre-split + pool's live objects onto its child release (re-annotating + meta.helm.sh/release-name, pinning helm.sh/resource-policy=keep) as a + pre-upgrade hook, BEFORE this control-plane-only chart reconciles. If + that migration did not run (its version slot was already stamped, or it + was renumbered out on a merge/backport), a live worker MachineDeployment + is still annotated to THIS parent release, and Helm's control-plane-only + re-render would prune it and cascade-delete the tenant's worker VMs. + Refuse to render instead: the upgrade blocks with a legible reason and + the workers keep running on the prior release until the migration runs. + + Fail-closed like system/seaweedfs naming-guard: the release Namespace is + a canary that always exists for a live install/upgrade, so an empty + canary means a client-side render (helm template / unittest / CI lint, + where lookup returns nil) that never prunes live objects — skip there. + Discriminate on the adoption annotation, never on mere existence: the + MachineDeployment always exists at render time, so an existence-only + guard would deadlock the routine upgrade (see helmreleases/ouroboros.yaml). */}} +{{- if .Release.IsUpgrade }} +{{- if lookup "v1" "Namespace" "" .Release.Namespace }} +{{- $unadopted := list }} +{{- range $md := (lookup "cluster.x-k8s.io/v1beta1" "MachineDeployment" $.Release.Namespace "").items }} +{{- if and (eq (dig "spec" "clusterName" "" $md) $.Release.Name) (eq (dig "metadata" "annotations" "meta.helm.sh/release-name" "" $md) $.Release.Name) }} +{{- $unadopted = append $unadopted (dig "metadata" "name" "" $md) }} +{{- end }} +{{- end }} +{{- if $unadopted }} +{{- fail (printf "kubernetes: refusing to render cluster %q control-plane-only — worker MachineDeployment(s) %v are still owned by this release, so the Phase 2 worker-pool adoption migration has not run. Rendering now would let Helm prune them and delete the tenant's worker VMs. Run the platform migration that adopts worker pools into KubernetesNodes (its slot must be stamped in the cozystack-version ConfigMap) before upgrading this cluster." $.Release.Name $unadopted) }} +{{- end }} +{{- end }} +{{- end }} {{- /* Validate user-supplied apiServer.extraVolumes / extraVolumeMounts before they are wired into the control-plane Deployment and apiserver container. diff --git a/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml b/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml new file mode 100644 index 000000000..57e5c2bf9 --- /dev/null +++ b/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml @@ -0,0 +1,30 @@ +suite: Phase 2 worker-pool adoption guard + +# The guard in templates/cluster.yaml refuses to render the control-plane-only +# chart when a live, un-adopted worker MachineDeployment for this cluster still +# exists (the adoption migration did not run), so Helm cannot silently prune it +# and delete the tenant's worker VMs. It keys on a live `lookup`, which returns +# nil under helm-unittest / `helm template` / any client-side render — so it MUST +# be a no-op there, even on an upgrade render. This suite pins that inertness: +# were the guard to fail closed on the empty offline lookup, the whole chart +# would stop rendering here and every parent-chart unit test would break. +release: + name: kubernetes-test + namespace: tenant-test + upgrade: true + +values: + - values/common.yaml + +templates: + - templates/cluster.yaml + +tests: + - it: is inert on a client-side upgrade render (empty lookup) and still renders the control plane + asserts: + # Render succeeded (the guard did not `fail`) and produced the control plane. + - isKind: + of: KamajiControlPlane + documentSelector: + path: kind + value: KamajiControlPlane