From fd14126beadfec5c1bf4570fc5706147d5abdeae Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Thu, 16 Jul 2026 00:27:56 +0300 Subject: [PATCH] feat(kubernetes-nodes): add worker node pool chart Introduce the kubernetes-nodes application chart: one HelmRelease per worker node pool, attached to a parent Kubernetes cluster by name convention (kubernetes-). Renders the KubevirtMachineTemplate, MachineDeployment, MachineHealthCheck, worker WorkloadMonitor and the talos-reconcile Job with its RBAC, extracted from the monolithic kubernetes chart. Pool objects are byte-identical to the parent's render for the same group, so a later adoption never churns live worker VMs. This is the additive half of the Phase 2 split (community#8); the parent kubernetes chart is left unchanged. Signed-off-by: Ivan Okhotnikov --- packages/apps/kubernetes-nodes/.helmignore | 5 + packages/apps/kubernetes-nodes/Chart.yaml | 7 + packages/apps/kubernetes-nodes/Makefile | 19 + packages/apps/kubernetes-nodes/README.md | 79 +++ .../apps/kubernetes-nodes/charts/cozy-lib | 1 + .../apps/kubernetes-nodes/files/versions.yaml | 5 + .../apps/kubernetes-nodes/images/kubectl.tag | 1 + .../kubernetes-nodes/logos/kubernetes.svg | 10 + .../apps/kubernetes-nodes/templates/NOTES.txt | 12 + .../kubernetes-nodes/templates/_helpers.tpl | 84 +++ .../kubernetes-nodes/templates/_resources.tpl | 12 + .../kubernetes-nodes/templates/_versions.tpl | 7 + .../templates/dashboard-resourcemap.yaml | 25 + .../kubernetes-nodes/templates/nodegroup.yaml | 543 ++++++++++++++++++ .../templates/talos-reconcile-job.yaml | 423 ++++++++++++++ .../apps/kubernetes-nodes/values.schema.json | 215 +++++++ packages/apps/kubernetes-nodes/values.yaml | 108 ++++ 17 files changed, 1556 insertions(+) create mode 100644 packages/apps/kubernetes-nodes/.helmignore create mode 100644 packages/apps/kubernetes-nodes/Chart.yaml create mode 100644 packages/apps/kubernetes-nodes/Makefile create mode 100644 packages/apps/kubernetes-nodes/README.md create mode 120000 packages/apps/kubernetes-nodes/charts/cozy-lib create mode 100644 packages/apps/kubernetes-nodes/files/versions.yaml create mode 100644 packages/apps/kubernetes-nodes/images/kubectl.tag create mode 100644 packages/apps/kubernetes-nodes/logos/kubernetes.svg create mode 100644 packages/apps/kubernetes-nodes/templates/NOTES.txt create mode 100644 packages/apps/kubernetes-nodes/templates/_helpers.tpl create mode 100644 packages/apps/kubernetes-nodes/templates/_resources.tpl create mode 100644 packages/apps/kubernetes-nodes/templates/_versions.tpl create mode 100644 packages/apps/kubernetes-nodes/templates/dashboard-resourcemap.yaml create mode 100644 packages/apps/kubernetes-nodes/templates/nodegroup.yaml create mode 100644 packages/apps/kubernetes-nodes/templates/talos-reconcile-job.yaml create mode 100644 packages/apps/kubernetes-nodes/values.schema.json create mode 100644 packages/apps/kubernetes-nodes/values.yaml diff --git a/packages/apps/kubernetes-nodes/.helmignore b/packages/apps/kubernetes-nodes/.helmignore new file mode 100644 index 000000000..cdada6673 --- /dev/null +++ b/packages/apps/kubernetes-nodes/.helmignore @@ -0,0 +1,5 @@ +.helmignore +/logos +/Makefile +/hack +/images/*/* diff --git a/packages/apps/kubernetes-nodes/Chart.yaml b/packages/apps/kubernetes-nodes/Chart.yaml new file mode 100644 index 000000000..8460d4a67 --- /dev/null +++ b/packages/apps/kubernetes-nodes/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: kubernetes-nodes +description: Worker node pool for a managed Kubernetes cluster +icon: /logos/kubernetes.svg +type: application +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process +appVersion: 1.32.6 diff --git a/packages/apps/kubernetes-nodes/Makefile b/packages/apps/kubernetes-nodes/Makefile new file mode 100644 index 000000000..307067ecb --- /dev/null +++ b/packages/apps/kubernetes-nodes/Makefile @@ -0,0 +1,19 @@ +include ../../../hack/package.mk + +.PHONY: test generate update + +test: + helm unittest . + +generate: + cozyvalues-gen -m 'kubernetesnodes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetesnodes/types.go + ../../../hack/update-crd.sh + +# Keep the worker-image inputs (Talos<->Kubernetes support matrix and the +# kubectl.tag used by the talos-reconcile Job) in lockstep with the parent +# kubernetes chart. They are copied, not symlinked, so `helm package` bundles +# them; run this after the parent chart's `make update`. +update: + cp ../kubernetes/files/versions.yaml files/versions.yaml + cp ../kubernetes/images/kubectl.tag images/kubectl.tag + $(MAKE) generate diff --git a/packages/apps/kubernetes-nodes/README.md b/packages/apps/kubernetes-nodes/README.md new file mode 100644 index 000000000..106536da8 --- /dev/null +++ b/packages/apps/kubernetes-nodes/README.md @@ -0,0 +1,79 @@ +# Kubernetes Worker Node Pool + +`kubernetes-nodes` manages a single worker node pool for a managed Kubernetes +cluster. It is the worker half of the Kubernetes-app split (see +[cozystack/community#8](https://github.com/cozystack/community/pull/8)): the +control plane lives in the `kubernetes` chart, and each pool is a separate +`kubernetes-nodes` release attached to a parent cluster by name. + +A `KubernetesNodes` CR named `-` (e.g. `myk8s-md0`) provisions a +`MachineDeployment` + `KubevirtMachineTemplate` + `MachineHealthCheck` wired to +the parent cluster `kubernetes-` in the same namespace, plus a +`talos-reconcile` Job that produces the pool's `TalosConfigTemplate`. + +## Parameters + +### Linkage + +| Name | Description | Type | Value | +| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ----- | +| `cluster` | Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. | `string` | `""` | + + +### Common Parameters + +| Name | Description | Type | Value | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ------------ | +| `storageClass` | StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. | `string` | `replicated` | + + +### Pool Sizing + +| Name | Description | Type | Value | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `minReplicas` | Minimum number of replicas in this pool. Used as the cluster-autoscaler floor. | `int` | `0` | +| `maxReplicas` | Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling. | `int` | `10` | +| `instanceType` | Virtual machine instance type. | `string` | `u1.medium` | +| `diskSize` | System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. | `quantity` | `20Gi` | +| `roles` | List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller. | `[]string` | `[]` | +| `resources` | Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. | `object` | `{}` | +| `resources.cpu` | CPU available. | `quantity` | `""` | +| `resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | +| `kubelet` | Kubelet resource reservations for this pool. | `object` | `{}` | +| `kubelet.systemReservedMemory` | Memory reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | +| `kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | +| `kubelet.systemReservedCpu` | CPU reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | +| `kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | +| `kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | +| `kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | +| `maxUnhealthy` | Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. | `string` | `50%` | +| `nodeStartupTimeout` | Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). | `string` | `10m` | + + +### Kubernetes Version + +| Name | Description | Type | Value | +| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `version` | Kubernetes major.minor version the pool joins. Must match the parent cluster's version and satisfy the Talos<->Kubernetes support matrix against `talos.version`. | `string` | `v1.35` | + + +### Talos Worker Image + +| Name | Description | Type | Value | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------ | +| `talos` | Talos worker image configuration. Keep in sync with the parent cluster's `talos`. | `object` | `{}` | +| `talos.version` | Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. | `string` | `v1.13.6` | +| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` | +| `talos.imageFactoryURL` | Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. | `string` | `https://factory.talos.dev` | +| `talos.installerRepository` | OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. | `string` | `factory.talos.dev/installer` | + + +### Images + +| Name | Description | Type | Value | +| ---------------- | ---------------------------------------------------------------------------------------- | -------- | ----- | +| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` | +| `images.kubectl` | Image used by the talos-reconcile Job (kubectl). Empty falls back to images/kubectl.tag. | `string` | `""` | + diff --git a/packages/apps/kubernetes-nodes/charts/cozy-lib b/packages/apps/kubernetes-nodes/charts/cozy-lib new file mode 120000 index 000000000..e18135098 --- /dev/null +++ b/packages/apps/kubernetes-nodes/charts/cozy-lib @@ -0,0 +1 @@ +../../../library/cozy-lib \ No newline at end of file diff --git a/packages/apps/kubernetes-nodes/files/versions.yaml b/packages/apps/kubernetes-nodes/files/versions.yaml new file mode 100644 index 000000000..aa5252921 --- /dev/null +++ b/packages/apps/kubernetes-nodes/files/versions.yaml @@ -0,0 +1,5 @@ +"v1.35": "v1.35.6" +"v1.34": "v1.34.9" +"v1.33": "v1.33.13" +"v1.32": "v1.32.13" +"v1.31": "v1.31.14" diff --git a/packages/apps/kubernetes-nodes/images/kubectl.tag b/packages/apps/kubernetes-nodes/images/kubectl.tag new file mode 100644 index 000000000..80d1e903c --- /dev/null +++ b/packages/apps/kubernetes-nodes/images/kubectl.tag @@ -0,0 +1 @@ +docker.io/alpine/k8s:1.33.4@sha256:b0523f0a244ddc4c8e055aa335c040d3d78b3ead5528f4544395f7f9f69c7b68 diff --git a/packages/apps/kubernetes-nodes/logos/kubernetes.svg b/packages/apps/kubernetes-nodes/logos/kubernetes.svg new file mode 100644 index 000000000..ac4e9a640 --- /dev/null +++ b/packages/apps/kubernetes-nodes/logos/kubernetes.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/apps/kubernetes-nodes/templates/NOTES.txt b/packages/apps/kubernetes-nodes/templates/NOTES.txt new file mode 100644 index 000000000..897126b89 --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/NOTES.txt @@ -0,0 +1,12 @@ +Worker node pool "{{ include "kubernetes-nodes.groupName" . }}" for cluster "{{ .Values.cluster }}". + +This pool is a separate release from the control plane. Two operational notes: + +1. Deletion order. Deleting the parent Kubernetes cluster while this pool still + exists leaves the pool release trying to recreate CAPI objects against a + cluster that is being torn down. Delete the KubernetesNodes pools first, then + the parent Kubernetes cluster. + +2. Version skew. Keep this pool's `version` and `talos.*` in sync with the parent + Kubernetes cluster. A pool on a Kubernetes/Talos combination the parent does + not run can produce workers the control plane will not admit. diff --git a/packages/apps/kubernetes-nodes/templates/_helpers.tpl b/packages/apps/kubernetes-nodes/templates/_helpers.tpl new file mode 100644 index 000000000..109648248 --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/_helpers.tpl @@ -0,0 +1,84 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubernetes.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubernetes.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubernetes.labels" -}} +helm.sh/chart: {{ include "kubernetes.chart" . }} +{{ include "kubernetes.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubernetes.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubernetes.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +DNS domain used INSIDE the tenant cluster (kubelet --cluster-domain, +apiserver --service-cluster-ip-range FQDNs, CoreDNS authoritative zone). +Pinned to Kamaji's default `networkProfile.clusterDomain`. Kept identical to +the parent kubernetes chart so the worker machineconfig this chart applies +matches the control plane it joins. + +Distinct from .Values._cluster["cluster-domain"], which is the MANAGEMENT +cluster domain (e.g. cozy.local) where the Kamaji control plane lives. +*/}} +{{- define "kubernetes.tenantClusterDomain" -}} +cluster.local +{{- end }} + +{{/* +Reconstruct the parent CAPI cluster name from the linkage value. + +The pool attaches to the parent Kubernetes CR named .Values.cluster, whose +HelmRelease (and therefore CAPI Cluster / KamajiControlPlane / KubevirtCluster +and every worker object it owns) is `kubernetes-`. This chart does +NOT use its own .Release.Name for CAPI wiring — the pool lives in a separate +HelmRelease from the control plane, so every reference that the monolithic +chart made through $.Release.Name is reconstructed here as kubernetes- +instead. Linkage is by name convention (mirrors vm-instance -> vm-disk), not +ownerReference or lookup-gated render. +*/}} +{{- define "kubernetes-nodes.clusterName" -}} +{{- if not .Values.cluster -}} +{{- fail "kubernetes-nodes: .Values.cluster is required — set it to the parent Kubernetes CR name so the pool attaches to cluster kubernetes-" -}} +{{- end -}} +{{- printf "kubernetes-%s" .Values.cluster -}} +{{- end -}} + +{{/* +The node-group name for this pool, derived from the release name. + +A KubernetesNodes CR is named - and gets the release prefix +`kubernetes-nodes-`, so the release name is `kubernetes-nodes--`. +The group name is the suffix. Enforcing the `-` segment keeps +every rendered object named `kubernetes--` — byte-identical to +what the monolithic chart rendered for the same group — and prevents two +clusters in one namespace from colliding on a pool named e.g. `md0`. +*/}} +{{- define "kubernetes-nodes.groupName" -}} +{{- $prefix := printf "kubernetes-nodes-%s-" .Values.cluster -}} +{{- if not (hasPrefix $prefix .Release.Name) -}} +{{- fail (printf "kubernetes-nodes: release name %q must start with %q — name the KubernetesNodes CR - (cluster=%q)" .Release.Name $prefix .Values.cluster) -}} +{{- end -}} +{{- trimPrefix $prefix .Release.Name -}} +{{- end -}} diff --git a/packages/apps/kubernetes-nodes/templates/_resources.tpl b/packages/apps/kubernetes-nodes/templates/_resources.tpl new file mode 100644 index 000000000..bb518b9ee --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/_resources.tpl @@ -0,0 +1,12 @@ +{{/* +Convert a CPU quantity string (e.g. "100m", "0.5", "1", "4") to millicores. +Returns a number string: "100", "500", "1000", "4000". +*/}} +{{- define "kubernetes.cpuToMillicores" -}} +{{- $str := . | toString -}} +{{- if hasSuffix "m" $str -}} +{{- trimSuffix "m" $str -}} +{{- else -}} +{{- mulf ($str | float64) 1000.0 | int | toString -}} +{{- end -}} +{{- end -}} diff --git a/packages/apps/kubernetes-nodes/templates/_versions.tpl b/packages/apps/kubernetes-nodes/templates/_versions.tpl new file mode 100644 index 000000000..fcd4cee17 --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/_versions.tpl @@ -0,0 +1,7 @@ +{{- define "kubernetes.versionMap" }} +{{- $versionMap := .Files.Get "files/versions.yaml" | fromYaml }} +{{- if not (hasKey $versionMap .Values.version) }} + {{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }} +{{- end }} +{{- index $versionMap .Values.version }} +{{- end }} diff --git a/packages/apps/kubernetes-nodes/templates/dashboard-resourcemap.yaml b/packages/apps/kubernetes-nodes/templates/dashboard-resourcemap.yaml new file mode 100644 index 000000000..c438fe7b0 --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/dashboard-resourcemap.yaml @@ -0,0 +1,25 @@ +{{- $clusterName := include "kubernetes-nodes.clusterName" . }} +{{- $groupName := include "kubernetes-nodes.groupName" . }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-dashboard-resources +rules: +- apiGroups: + - cozystack.io + resources: + - workloadmonitors + resourceNames: + - {{ $clusterName }}-{{ $groupName }} + verbs: ["get", "list", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-dashboard-resources +subjects: +{{ include "cozy-lib.rbac.subjectsForTenantAndAccessLevel" (list "use" .Release.Namespace) }} +roleRef: + kind: Role + name: {{ .Release.Name }}-dashboard-resources + apiGroup: rbac.authorization.k8s.io diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml new file mode 100644 index 000000000..6d09f06e2 --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -0,0 +1,543 @@ +{{- /* + Worker node pool for a managed Kubernetes cluster. + + This chart is the second half of the Kubernetes-app split (cozystack/community#8): + the control plane (Cluster / KamajiControlPlane / KubevirtCluster / PKI) lives in + the `kubernetes` chart, and each worker pool is a separate `kubernetes-nodes` + release. The pool objects (KubevirtMachineTemplate, MachineDeployment, + MachineHealthCheck, worker WorkloadMonitor) and the talos-reconcile Job that + produces the pool's TalosConfigTemplate are rendered here, wired to the parent + cluster `kubernetes-` by name convention (see _helpers.tpl). Every + reference the monolithic chart made through $.Release.Name is reconstructed here + as the parent clusterName so the objects are byte-identical to what the parent + rendered for the same group — the KubevirtMachineTemplate is content-hash-named, + so any drift would roll every live worker VM. + + Talos -> Kubernetes support matrix. Each Talos minor release supports a bounded + window of Kubernetes minor versions; running a kubelet outside that window + produces a silently broken Talos+kubelet combination. Fail the render instead. + Source: https://docs.siderolabs.com/talos/v1.13/getting-started/support-matrix +*/}} +{{- $talosK8sSupportMatrix := dict + "v1.13" (list "v1.31" "v1.32" "v1.33" "v1.34" "v1.35" "v1.36") +}} +{{- $talosMinor := regexFind "^v[0-9]+\\.[0-9]+" .Values.talos.version }} +{{- with index $talosK8sSupportMatrix $talosMinor }} +{{- if not (has $.Values.version .) }} +{{- fail (printf "Kubernetes %s is not supported by Talos %s. Supported versions: %v. Bump 'version' or set talos.version explicitly." $.Values.version $.Values.talos.version .) }} +{{- end }} +{{- end }} +{{- $clusterName := include "kubernetes-nodes.clusterName" . }} +{{- $groupName := include "kubernetes-nodes.groupName" . }} +{{- $kubevirtmachinetemplateNames := list }} +{{- $group := dict + "minReplicas" .Values.minReplicas + "maxReplicas" .Values.maxReplicas + "instanceType" .Values.instanceType + "diskSize" .Values.diskSize + "storageClass" .Values.storageClass + "roles" .Values.roles + "resources" .Values.resources + "gpus" .Values.gpus + "kubelet" .Values.kubelet +}} +{{- define "kubevirtmachinetemplate" -}} +spec: + {{- /* Talos has no SSH; use 'none' so CAPK does not block Machine.Ready + on an SSH probe that will never succeed. The Machine reaches Ready + once kubelet registers with the tenant apiserver via standard CSR. */}} + virtualMachineBootstrapCheck: + checkStrategy: none + virtualMachineTemplate: + metadata: + namespace: {{ $.Release.Namespace }} + labels: + {{- range .group.roles }} + node-role.kubernetes.io/{{ . }}: "" + {{- end }} + spec: + {{- /* Explicit resources take precedence over instanceType. When a node + group is sized by resources (both cpu and memory set), the + instancetype is omitted from the VM: KubeVirt rejects a + VirtualMachine that references an instancetype and also overrides + either domain.cpu or domain.memory. */}} + {{- $sizedByResources := and .group.resources .group.resources.cpu .group.resources.memory }} + {{- if and .group.instanceType (not $sizedByResources) }} + instancetype: + kind: VirtualMachineClusterInstancetype + name: {{ .group.instanceType }} + {{- end }} + runStrategy: Always + dataVolumeTemplates: + - metadata: + name: disk-system + annotations: + cdi.kubevirt.io/storage.usePopulator: "false" + spec: + {{- /* Use the openstack/configdrive Talos image: KubeVirt CAPK + injects bootstrap data via cloudInitConfigDrive (OpenStack + metadata service shape), and the openstack platform image + is the one that reads it. The nocloud-amd64 image silently + ignores configdrive and never applies the machineconfig. */}} + source: + http: + url: {{ $.Values.talos.imageFactoryURL | trimSuffix "/" }}/image/{{ $.Values.talos.schematicID }}/{{ $.Values.talos.version }}/openstack-amd64.raw.xz + {{- if hasKey .group "ephemeralStorage" }} + {{- fail (printf "nodeGroup %q: ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system." .groupName) }} + {{- end }} + storage: + resources: + requests: + storage: {{ .group.diskSize | default "20Gi" | quote }} + {{- /* + Worker VMs live-migrate, so their disks need ReadWriteMany. The + RWX access mode is supplied by the chosen StorageClass's CDI + StorageProfile — it is not set on the DataVolume here. linstor-csi + rejects RWX volumes that are not on a DRBD-backed StorageClass, so + an empty nodeGroup storageClass falls back to the application-level + storageClass (a replicated/DRBD class) instead of the cluster + default, which may be a non-DRBD class that the CSI driver would + reject. Both empty still omits the field so the cluster default + applies. + */}} + {{- with .group.storageClass | default $.Values.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + template: + metadata: + annotations: + kubevirt.io/allow-pod-bridge-network-live-migration: "true" + labels: + apps.cozystack.io/application.group: apps.cozystack.io + apps.cozystack.io/application.kind: Kubernetes + apps.cozystack.io/application.name: {{ .clusterName | trimPrefix "kubernetes-" | quote }} + {{- range .group.roles }} + node-role.kubernetes.io/{{ . }}: "" + {{- end }} + cluster.x-k8s.io/deployment-name: {{ .clusterName }}-{{ .groupName }} + spec: + {{- if .Values._cluster.scheduling }} + {{- $rawConstraints := get .Values._cluster.scheduling "globalAppTopologySpreadConstraints" }} + {{- if $rawConstraints }} + {{- $rawConstraints | fromYaml | toYaml | nindent 10 }} + labelSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ .clusterName }} + {{- end }} + {{- end }} + domain: + {{- if and .group.resources .group.resources.cpu }} + cpu: + threads: 1 + cores: {{ .group.resources.cpu }} + sockets: 1 + {{- end }} + devices: + {{- if .group.gpus }} + gpus: + {{- range $i, $gpu := .group.gpus }} + - name: gpu{{ add $i 1 }} + deviceName: {{ $gpu.name }} + {{- end }} + {{- end }} + disks: + - name: system + disk: + bus: virtio + pciAddress: 0000:07:00.0 + blockSize: + custom: + logical: 512 + physical: 4096 + interfaces: + - name: default + bridge: {} + {{- if and .group.resources .group.resources.memory }} + memory: + guest: {{ .group.resources.memory }} + {{- end }} + resources: + {{/* 2Gi is enough: persistent DataVolumes are not counted as pod ephemeral storage */}} + {{- include "cozy-lib.resources.sanitize" (list (dict "ephemeral-storage" "2Gi") $) | nindent 14 }} + evictionStrategy: External + volumes: + - name: system + dataVolume: + name: disk-system + networks: + - name: default + pod: {} +{{- end }} +{{- /* instanceType and explicit resources are alternative sizing sources, and + explicit resources win. KubeVirt rejects a VirtualMachine that references + an instancetype while also overriding either domain.cpu or domain.memory, + so when a group is sized by resources (both cpu and memory set) the + instancetype is omitted from the VM (see the VM spec above) and the lookup + below is skipped — a stale instancetype name cannot fail an otherwise + valid, resource-sized group. Explicit resources are all-or-nothing: a + partial spec (only one of cpu/memory) would emit one domain override + alongside the instancetype, which KubeVirt rejects, so it fails here with + a clear render-time error instead of a late admission rejection. */}} +{{- $hasCpu := and $group.resources $group.resources.cpu }} +{{- $hasMem := and $group.resources $group.resources.memory }} +{{- if and $group.instanceType (or (and $hasCpu (not $hasMem)) (and $hasMem (not $hasCpu))) }} +{{- fail (printf "nodeGroup %s: set both resources.cpu and resources.memory, or neither" $groupName) }} +{{- end }} +{{/* Resolve instanceType once per group — reused for kubelet reservations and capacity annotations. */}} +{{- $instanceType := dict }} +{{- if and $group.instanceType (not (and $hasCpu $hasMem)) }} +{{- $instanceType = (lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType) }} +{{- if not $instanceType }} +{{- fail (printf "nodeGroup %s: specified instanceType %q not found in cluster" $groupName $group.instanceType) }} +{{- end }} +{{- end }} +{{/* Compute kubelet reservations: explicit resources.memory/cpu wins over instanceType spec. + Auto-computed system/kube reserved memory = 5% of effective memory, clamped to [256Mi, 1Gi]. + Auto-computed system/kube reserved CPU = 5% of effective CPU, clamped to [50m, 500m]. */}} +{{- $effectiveMemory := "" }} +{{- if and $group.resources $group.resources.memory }} +{{- $effectiveMemory = $group.resources.memory | toString }} +{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} +{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} +{{- end }} +{{- $effectiveCpu := "" }} +{{- if and $group.resources $group.resources.cpu }} +{{- $effectiveCpu = $group.resources.cpu | toString }} +{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} +{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} +{{- end }} +{{- $autoReservedMi := 256 }} +{{- if $effectiveMemory }} +{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} +{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} +{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} +{{- end }} +{{- $autoReservedMillicores := 50 }} +{{- if $effectiveCpu }} +{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} +{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }} +{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} +{{- end }} +{{- $kubeletOverride := $group.kubelet | default dict }} +{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} +{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} +{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} +{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} +{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} +{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }} +{{/* Validate reservation field formats (user-supplied values only; computed defaults are valid by construction). */}} +{{- $validations := list + (dict "key" "systemReservedMemory" "value" $kubeletOverride.systemReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)") + (dict "key" "kubeReservedMemory" "value" $kubeletOverride.kubeReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)") + (dict "key" "systemReservedCpu" "value" $kubeletOverride.systemReservedCpu "pattern" `^([0-9]+m|[0-9]+(\.[0-9]+)?)$` "hint" "a valid Kubernetes CPU quantity (integer millicores like 100m, or decimal cores like 0.5, 1)") + (dict "key" "kubeReservedCpu" "value" $kubeletOverride.kubeReservedCpu "pattern" `^([0-9]+m|[0-9]+(\.[0-9]+)?)$` "hint" "a valid Kubernetes CPU quantity (integer millicores like 100m, or decimal cores like 0.5, 1)") +}} +{{- range $validations }} +{{- if and .value (not (regexMatch .pattern (.value | toString))) }} +{{- fail (printf "nodeGroup %s: invalid %s value %q — must be %s" $groupName .key (.value | toString) .hint) }} +{{- end }} +{{- end }} +{{/* Validate that user-supplied reservation values are positive (non-zero) */}} +{{- range $_, $entry := list (dict "key" "systemReservedMemory" "value" $kubeletOverride.systemReservedMemory) (dict "key" "kubeReservedMemory" "value" $kubeletOverride.kubeReservedMemory) }} +{{- if $entry.value }} +{{- $bytes := include "cozy-lib.resources.toFloat" ($entry.value | toString) | float64 }} +{{- if le $bytes 0.0 }} +{{- fail (printf "nodeGroup %s: %s must be a positive value, got %s" $groupName $entry.key ($entry.value | toString)) }} +{{- end }} +{{- end }} +{{- end }} +{{- range $_, $entry := list (dict "key" "systemReservedCpu" "value" $kubeletOverride.systemReservedCpu) (dict "key" "kubeReservedCpu" "value" $kubeletOverride.kubeReservedCpu) }} +{{- if $entry.value }} +{{- $cpuMillis := include "kubernetes.cpuToMillicores" $entry.value | float64 }} +{{- if le $cpuMillis 0.0 }} +{{- fail (printf "nodeGroup %s: %s must be a positive value, got %s" $groupName $entry.key ($entry.value | toString)) }} +{{- end }} +{{- end }} +{{- end }} +{{/* Validate absolute eviction threshold formats before converting */}} +{{- if not (hasSuffix "%" $evictionHardMemory) }} +{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` $evictionHardMemory) }} +{{- fail (printf "nodeGroup %s: invalid evictionHardMemory value %q — must be a percentage (e.g. 7%%) or a valid Kubernetes memory quantity (e.g. 500Mi, 1Gi, 1e9)" $groupName $evictionHardMemory) }} +{{- end }} +{{- end }} +{{- if not (hasSuffix "%" $evictionSoftMemory) }} +{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` $evictionSoftMemory) }} +{{- fail (printf "nodeGroup %s: invalid evictionSoftMemory value %q — must be a percentage (e.g. 10%%) or a valid Kubernetes memory quantity (e.g. 500Mi, 1Gi, 1e9)" $groupName $evictionSoftMemory) }} +{{- end }} +{{- end }} +{{- if and (hasSuffix "%" $evictionHardMemory) (hasSuffix "%" $evictionSoftMemory) }} +{{- $hardPctStr := trimSuffix "%" $evictionHardMemory }} +{{- $softPctStr := trimSuffix "%" $evictionSoftMemory }} +{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $hardPctStr) }} +{{- fail (printf "nodeGroup %s: evictionHardMemory percentage value %q is not a valid number" $groupName $hardPctStr) }} +{{- end }} +{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $softPctStr) }} +{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage value %q is not a valid number" $groupName $softPctStr) }} +{{- end }} +{{- $hardPct := $hardPctStr | float64 }} +{{- $softPct := $softPctStr | float64 }} +{{- if le $hardPct 0.0 }} +{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $hardPctStr) }} +{{- end }} +{{- if gt $hardPct 100.0 }} +{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $hardPctStr) }} +{{- end }} +{{- if le $softPct 0.0 }} +{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $softPctStr) }} +{{- end }} +{{- if gt $softPct 100.0 }} +{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $softPctStr) }} +{{- end }} +{{- if ge $hardPct $softPct }} +{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) must be strictly less than evictionSoftMemory (%s)" $groupName $evictionHardMemory $evictionSoftMemory) }} +{{- end }} +{{- else if and (not (hasSuffix "%" $evictionHardMemory)) (not (hasSuffix "%" $evictionSoftMemory)) }} +{{- $hardBytes := include "cozy-lib.resources.toFloat" $evictionHardMemory | float64 }} +{{- $softBytes := include "cozy-lib.resources.toFloat" $evictionSoftMemory | float64 }} +{{- if ge $hardBytes $softBytes }} +{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) must be strictly less than evictionSoftMemory (%s)" $groupName $evictionHardMemory $evictionSoftMemory) }} +{{- end }} +{{- else }} +{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) and evictionSoftMemory (%s) must use the same unit type (both percentage or both absolute)" $groupName $evictionHardMemory $evictionSoftMemory) }} +{{- end }} +{{/* Resolve eviction hard threshold to bytes for arithmetic. + Percentage is relative to effective node memory; absolute values are used as-is. */}} +{{- $evictionHardBytes := 0.0 }} +{{- if $effectiveMemory }} +{{- if hasSuffix "%" $evictionHardMemory }} +{{- $evictionPct := trimSuffix "%" $evictionHardMemory | float64 }} +{{- $evictionHardBytes = mulf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) (divf $evictionPct 100.0) }} +{{- else }} +{{- $evictionHardBytes = include "cozy-lib.resources.toFloat" $evictionHardMemory | float64 }} +{{- end }} +{{- end }} +{{/* Validate that total reserved memory (system + kube + eviction) does not exceed effective node memory */}} +{{- if $effectiveMemory }} +{{- $sysBytes := include "cozy-lib.resources.toFloat" $systemReservedMemory | float64 }} +{{- $kubeBytes := include "cozy-lib.resources.toFloat" $kubeReservedMemory | float64 }} +{{- $totalReservedBytes := addf (addf $sysBytes $kubeBytes) $evictionHardBytes }} +{{- $effectiveMemBytes := include "cozy-lib.resources.toFloat" $effectiveMemory | float64 }} +{{- if ge $totalReservedBytes $effectiveMemBytes }} +{{- fail (printf "nodeGroup %s: total reserved memory (systemReservedMemory=%s + kubeReservedMemory=%s + evictionHardMemory=%s) must be less than effective node memory (%s)" $groupName $systemReservedMemory $kubeReservedMemory $evictionHardMemory $effectiveMemory) }} +{{- end }} +{{- end }} +{{/* Validate that total reserved CPU does not exceed effective node CPU */}} +{{- if $effectiveCpu }} +{{- $sysCpuMillis := include "kubernetes.cpuToMillicores" $systemReservedCpu | float64 }} +{{- $kubeCpuMillis := include "kubernetes.cpuToMillicores" $kubeReservedCpu | float64 }} +{{- $totalCpuMillis := addf $sysCpuMillis $kubeCpuMillis }} +{{- $effectiveCpuMillicoresV := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} +{{- $effectiveCpuMillis := float64 $effectiveCpuMillicoresV }} +{{- if ge $totalCpuMillis $effectiveCpuMillis }} +{{- fail (printf "nodeGroup %s: total reserved CPU (systemReservedCpu=%s + kubeReservedCpu=%s) must be less than effective node CPU (%s cores)" $groupName $systemReservedCpu $kubeReservedCpu $effectiveCpu) }} +{{- end }} +{{- end }} +{{/* Guard: explicit kubelet overrides require effective memory for validation */}} +{{- if not $effectiveMemory }} +{{- $hasOverride := or $kubeletOverride.systemReservedMemory $kubeletOverride.kubeReservedMemory $kubeletOverride.evictionHardMemory $kubeletOverride.evictionSoftMemory }} +{{- if $hasOverride }} +{{- fail (printf "nodeGroup %s: kubelet memory reservations require effective memory to validate — set resources.memory or instanceType" $groupName) }} +{{- end }} +{{- end }} +{{/* Guard: explicit kubelet CPU overrides require effective CPU for validation */}} +{{- if not $effectiveCpu }} +{{- $hasCpuOverride := or $kubeletOverride.systemReservedCpu $kubeletOverride.kubeReservedCpu }} +{{- if $hasCpuOverride }} +{{- fail (printf "nodeGroup %s: cannot validate kubelet CPU reservations without effective CPU — set resources.cpu or a valid instanceType" $groupName) }} +{{- end }} +{{- end }} +--- +{{- /* TalosConfigTemplate is not rendered here — the chart cannot produce a + valid Talos worker machineconfig at Helm template time because three of + the inputs (apiserver Service ClusterIP, Talos CA, Kamaji-issued tenant + CA) are created asynchronously. The main-phase Job in + templates/talos-reconcile-job.yaml waits for those inputs and produces + the TalosConfigTemplate with a KamajiControlPlane ownerReference for + natural GC. The MachineDeployment below references the TCT name; CAPI's + Machine controller blocks on its absence and unblocks the moment the Job + applies it. */}} +--- +{{- $context := deepCopy $ }} +{{- $_ := set $context "group" $group }} +{{- $_ := set $context "groupName" $groupName }} +{{- $_ := set $context "clusterName" $clusterName }} +{{- $kubevirtmachinetemplate := include "kubevirtmachinetemplate" $context }} +{{- $kubevirtmachinetemplateHash := $kubevirtmachinetemplate | sha256sum | trunc 6 }} +{{- $kubevirtmachinetemplateName := printf "%s-%s-%s" $clusterName $groupName $kubevirtmachinetemplateHash }} +{{- $kubevirtmachinetemplateNames = append $kubevirtmachinetemplateNames $kubevirtmachinetemplateName }} +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: KubevirtMachineTemplate +metadata: + name: {{ $clusterName }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }} + namespace: {{ $.Release.Namespace }} +spec: + template: + {{- $kubevirtmachinetemplate | nindent 4 }} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: {{ $clusterName }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "{{ $group.minReplicas }}" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "{{ $group.maxReplicas }}" + {{- if $effectiveMemory }} + {{- $sysBytes := include "cozy-lib.resources.toFloat" $systemReservedMemory | float64 }} + {{- $kubeBytes := include "cozy-lib.resources.toFloat" $kubeReservedMemory | float64 }} + {{- $totalReservedBytes := addf (addf $sysBytes $kubeBytes) $evictionHardBytes }} + {{- $allocatableBytes := subf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) $totalReservedBytes }} + {{- $allocatableMi := divf $allocatableBytes 1048576.0 | floor }} + capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $allocatableMi }}Mi" + {{- end }} + {{- if $effectiveCpu }} + {{- $sysCpuMillisA := include "kubernetes.cpuToMillicores" $systemReservedCpu | float64 }} + {{- $kubeCpuMillisA := include "kubernetes.cpuToMillicores" $kubeReservedCpu | float64 }} + {{- $effectiveCpuMillicoresA := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} + {{- $effectiveCpuMillisA := float64 $effectiveCpuMillicoresA }} + {{- $allocatableCpuMillis := subf $effectiveCpuMillisA (addf $sysCpuMillisA $kubeCpuMillisA) }} + {{- $allocatableCpuMillisInt := $allocatableCpuMillis | ceil | int }} + capacity.cluster-autoscaler.kubernetes.io/cpu: "{{ $allocatableCpuMillisInt }}m" + {{- end }} +spec: + clusterName: {{ $clusterName }} + # `spec.replicas` is intentionally omitted so the field is owned by + # the cluster-autoscaler alone. Two independent CAPI mechanisms — + # not Server-Side-Apply field managers — carry the guarantee: + # 1. Create: the MachineDeployment defaulting webhook seeds + # `spec.replicas` from the `cluster-api-autoscaler-node-group- + # min-size` annotation (see + # https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling). + # 2. Update: `calculateMachineDeploymentReplicas` preserves the + # existing live value (`oldMD.Spec.Replicas`) whenever the + # incoming spec has `replicas: nil`, clamping into + # `[min, max]` if it drifts outside — so a cluster the + # autoscaler holds above min-size is not drained by a + # `helm upgrade` that would otherwise re-render this field. + # Flux helm-controller applies this chart through the Helm SDK + # three-way merge (not SSA), so field-manager ownership is not the + # mechanism at play. The chart guarantee reduces to "leave the + # field out of the rendered manifest and let CAPI's update-time + # defaulting keep the live value stable". Emitting a chart-managed + # `replicas: N` here (as this chart did before + # cozystack/cozystack#3231) drains workers on every `helm upgrade` + # action until the autoscaler notices unschedulable Pods and + # scales back up — a real disruption window on every routine + # platform bump. + strategy: + rollingUpdate: + maxSurge: {{ $group.maxReplicas }} + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + cluster.x-k8s.io/deployment-name: {{ $clusterName }}-{{ $groupName }} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + cluster.x-k8s.io/deployment-name: {{ $clusterName }}-{{ $groupName }} + {{- range $group.roles }} + node-role.kubernetes.io/{{ . }}: "" + {{- end }} + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: TalosConfigTemplate + name: {{ $clusterName }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} + clusterName: {{ $clusterName }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: KubevirtMachineTemplate + name: {{ $clusterName }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }} + namespace: {{ $.Release.Namespace }} + version: {{ include "kubernetes.versionMap" $}} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: {{ $clusterName }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} +spec: + clusterName: {{ $clusterName }} + {{- /* MHC spec.maxUnhealthy is an IntOrString. CAPI admission accepts a + bare integer (0, 1, ...) or a percentage string ("0%", "50%"), but + rejects a quoted plain integer ("0"). Quote only when the operator + supplied a percentage; otherwise render the value as a bare int so + the admission webhook is happy with the chart-default "0". */}} + {{- $maxUnhealthy := toString $.Values.maxUnhealthy }} + {{- $muSource := "maxUnhealthy" }} + {{- if hasSuffix "%" $maxUnhealthy }} + {{- if not (regexMatch "^[0-9]+%$" $maxUnhealthy) }} + {{- fail (printf "%s must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $muSource $maxUnhealthy) }} + {{- end }} + maxUnhealthy: {{ $maxUnhealthy | quote }} + {{- else if regexMatch "^[0-9]+$" $maxUnhealthy }} + maxUnhealthy: {{ $maxUnhealthy | int }} + {{- else }} + {{- fail (printf "%s must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $muSource $maxUnhealthy) }} + {{- end }} + nodeStartupTimeout: {{ $.Values.nodeStartupTimeout }} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + cluster.x-k8s.io/deployment-name: {{ $clusterName }}-{{ $groupName }} + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 30s + - type: Ready + status: "False" + timeout: 300s +--- +apiVersion: cozystack.io/v1alpha1 +kind: WorkloadMonitor +metadata: + name: {{ $clusterName }}-{{ $groupName }} + namespace: {{ $.Release.Namespace }} +spec: + minReplicas: {{ $group.minReplicas }} + kind: kubernetes + type: worker + selector: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + cluster.x-k8s.io/deployment-name: {{ $clusterName }}-{{ $groupName }} + cluster.x-k8s.io/role: worker + version: {{ $.Chart.Version }} +--- +{{- /* +We must preserve all previous KubevirtMachineTemplates for THIS pool until a +MachineSet stops referencing them. Scoped to this pool's own hash-named KMTs +(prefix kubernetes---) so sibling kubernetes-nodes releases in +the same namespace never re-emit and fight over each other's templates. +*/ -}} +{{- $mss := (lookup "cluster.x-k8s.io/v1beta1" "MachineSet" $.Release.Namespace "").items }} +{{- $kmtPrefix := printf "%s-%s-" $clusterName $groupName }} +{{- $oldKubevirtmachinetemplates := dict }} +{{- range $kmt := (lookup "infrastructure.cluster.x-k8s.io/v1alpha1" "KubevirtMachineTemplate" $.Release.Namespace "").items }} +{{- range $or := $kmt.metadata.ownerReferences }} +{{- if and (eq $or.kind "Cluster") (eq $or.name $clusterName) (hasPrefix $kmtPrefix $kmt.metadata.name) }} +{{- range $ms := $mss }} +{{- if and (eq $ms.spec.template.spec.infrastructureRef.kind "KubevirtMachineTemplate") (eq $ms.spec.template.spec.infrastructureRef.name $kmt.metadata.name) }} +{{- if not (has $kmt.metadata.name $kubevirtmachinetemplateNames) }} +{{- $oldKubevirtmachinetemplates = merge $oldKubevirtmachinetemplates (dict $kmt.metadata.name $kmt) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $oldKubevirtmachinetemplates }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: KubevirtMachineTemplate +metadata: + name: {{ .metadata.name }} + namespace: {{ .metadata.namespace }} +spec: + {{- .spec | toYaml | nindent 2 }} +{{- end }} diff --git a/packages/apps/kubernetes-nodes/templates/talos-reconcile-job.yaml b/packages/apps/kubernetes-nodes/templates/talos-reconcile-job.yaml new file mode 100644 index 000000000..ccd1be672 --- /dev/null +++ b/packages/apps/kubernetes-nodes/templates/talos-reconcile-job.yaml @@ -0,0 +1,423 @@ +{{- /* + Main-phase Job (NOT a Helm hook) that produces this pool's TalosConfigTemplate + and patches the parent KamajiControlPlane.spec.network.certSANs with the live + apiserver Service ClusterIP. + + The Talos worker machineconfig needs values that only exist AFTER the parent + control-plane release has applied: the apiserver Service ClusterIP (emitted by + Kamaji), the Talos-CA Secret (issued by cert-manager from the parent's Talos + PKI), and the Kubernetes-CA Secret (generated by Kamaji). A Helm `lookup` for + any of these races on every fresh install, so this Job blocks until all four + inputs are observably present, then `kubectl apply`s the TalosConfigTemplate + (idempotent) with an ownerReference to the parent KamajiControlPlane so it is + garbage-collected on the parent's uninstall. + + Cross-release ordering. The Job reads objects owned by the PARENT `kubernetes` + release (Service, Secrets, KamajiControlPlane named kubernetes-). No + hard per-tenant dependsOn is possible across the two HelmReleases, so ordering + is handled at runtime by this Job's ~10-minute input-wait loop plus the + release.cozystack.io/helm-install-disable-wait annotation on the pool's + ApplicationDefinition. + + Churn-free across upgrades. The Job is immutable, so it carries a content-hash + name suffix; an identical render yields the same name and Helm no-ops it. The + applied TalosConfigTemplate keeps a stable name (kubernetes--, + referenced by MachineDeployment.bootstrap.configRef), so an unchanged render + re-applies a byte-identical TCT — a CAPI no-op that never rolls workers. +*/}} +{{- $clusterName := include "kubernetes-nodes.clusterName" . }} +{{- $groupName := include "kubernetes-nodes.groupName" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-talos-reconcile + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-talos-reconcile + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["services"] + resourceNames: ["{{ $clusterName }}"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: + - "{{ $clusterName }}-talos-secrets" + - "{{ $clusterName }}-talos-ca" + - "{{ $clusterName }}-ca" + verbs: ["get"] + - apiGroups: ["controlplane.cluster.x-k8s.io"] + resources: ["kamajicontrolplanes"] + resourceNames: ["{{ $clusterName }}"] + verbs: ["get", "patch"] + - apiGroups: ["bootstrap.cluster.x-k8s.io"] + resources: ["talosconfigtemplates"] + verbs: ["get", "create", "patch", "update"] +--- +{{- /* The tenant-root chart installs a default-deny egress policy on every + pod in the tenant namespace; without an explicit allow, this Job + cannot reach the management-cluster apiserver to read the Service + ClusterIP / Secrets / KamajiControlPlane it has to reconcile. Cilium + policies are additive, so this rule layers on top of the tenant + default-deny without weakening it for anything else. Endpoint + selector is scoped to pods carrying the reconcile label only. */}} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ .Release.Name }}-talos-reconcile + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + cozystack.io/talos-reconcile: "{{ .Release.Name }}" + egress: + - toEntities: + - kube-apiserver + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + "k8s:k8s-app": kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-talos-reconcile + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-talos-reconcile +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-talos-reconcile + namespace: {{ .Release.Namespace }} +--- +{{- /* Cluster-scoped read on the management CoreDNS Service so the Job + can look up its ClusterIP at runtime and inject it into the worker + machineconfig as a host-level nameserver. resourceNames pins the + grant to the single well-known Service name kube-dns. */}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["services"] + resourceNames: ["kube-dns"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-talos-reconcile + namespace: {{ .Release.Namespace }} +{{- /* Job spec rendered through a named template so the Job name can carry + a content hash of the whole spec. $.Release is the pool release (used for + the SA it runs as and its network-policy label); .clusterName targets the + parent control-plane objects (Service/Secrets/KCP/TCT). */}} +{{- define "kubernetes-nodes.talos-reconcile-job-spec" -}} +spec: + backoffLimit: 30 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + cozystack.io/talos-reconcile: {{ $.Release.Name | quote }} + spec: + restartPolicy: OnFailure + serviceAccountName: {{ $.Release.Name }}-talos-reconcile + containers: + - name: kubectl + image: "{{ .kubectlImage }}" + env: + - name: RELEASE + value: {{ .clusterName | quote }} + - name: NS + value: {{ $.Release.Namespace | quote }} + - name: GROUP_NAME + value: {{ .groupName | quote }} + - name: TALOS_TOKEN + valueFrom: + secretKeyRef: + name: {{ .clusterName }}-talos-secrets + key: token + - name: CLUSTER_ID + valueFrom: + secretKeyRef: + name: {{ .clusterName }}-talos-secrets + key: clusterId + - name: CLUSTER_SECRET + valueFrom: + secretKeyRef: + name: {{ .clusterName }}-talos-secrets + key: clusterSecret + - name: BOOTSTRAP_TOKEN + valueFrom: + secretKeyRef: + name: {{ .clusterName }}-talos-secrets + key: bootstrapToken + command: + - sh + - -ec + - | + echo "[$(date -u +%H:%M:%S)] talos-reconcile/${GROUP_NAME}: waiting for Service ClusterIP, Talos CA, K8s CA, management CoreDNS" + for i in $(seq 1 120); do + SVC_IP=$(kubectl -n "$NS" get svc "$RELEASE" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true) + TALOS_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-talos-ca" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) + K8S_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true) + COREDNS_IP=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true) + if [ -n "$SVC_IP" ] && [ -n "$TALOS_CA_B64" ] && [ -n "$K8S_CA_B64" ] && [ -n "$COREDNS_IP" ]; then + break + fi + sleep 5 + done + if [ -z "$SVC_IP" ] || [ -z "$TALOS_CA_B64" ] || [ -z "$K8S_CA_B64" ] || [ -z "$COREDNS_IP" ]; then + echo "[$(date -u +%H:%M:%S)] timed out waiting for lookup inputs (svc='$SVC_IP' talos-ca=$([ -n "$TALOS_CA_B64" ] && echo ok) k8s-ca=$([ -n "$K8S_CA_B64" ] && echo ok) coredns='$COREDNS_IP')" + exit 1 + fi + echo "[$(date -u +%H:%M:%S)] inputs ready (svc=${SVC_IP} coredns=${COREDNS_IP})" + + # Patch KamajiControlPlane.spec.network.certSANs so the apiserver + # TLS cert covers the live Service ClusterIP that Talos workers + # dial. JSON-merge-patch is array-replace semantics, which is what + # we want here — Kamaji owns the full SAN list. + kubectl -n "$NS" patch kamajicontrolplane "$RELEASE" --type=merge --patch "$( + cat <`; the KubernetesNodes CR must be named `-`. +## @x-cozystack-options {source: kubernetes} +cluster: "" + +## +## @section Common Parameters +## + +## @param {string} storageClass - StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. +## @x-cozystack-options {source: storageclass} +storageClass: replicated + +## +## @section Pool Sizing +## + +## @typedef {struct} Resources - Explicit CPU and memory configuration for a node. +## @field {quantity} [cpu] - CPU available. +## @field {quantity} [memory] - Memory (RAM) available. + +## @typedef {struct} GPU - GPU configuration. +## @field {string} name - Name of GPU, such as "nvidia.com/AD102GL_L40S". +## @x-cozystack-options {source: gpu} + +## @typedef {struct} Kubelet - Kubelet resource reservations for worker nodes. +## @field {string} [systemReservedMemory] - Memory reserved for host OS. Auto-computed from instanceType if empty. +## @field {string} [kubeReservedMemory] - Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. +## @field {string} [systemReservedCpu] - CPU reserved for host OS. Auto-computed from instanceType if empty. +## @field {string} [kubeReservedCpu] - CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. +## @field {string} [evictionHardMemory]="7%" - Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). +## @field {string} [evictionSoftMemory]="10%" - Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). + +## @param {int} minReplicas=0 - Minimum number of replicas in this pool. Used as the cluster-autoscaler floor. +minReplicas: 0 + +## @param {int} maxReplicas=10 - Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling. +maxReplicas: 10 + +## @param {string} instanceType="u1.medium" - Virtual machine instance type. +## @x-cozystack-options {source: instancetype} +instanceType: u1.medium + +## @param {quantity} diskSize="20Gi" - System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. +diskSize: 20Gi + +## @param {[]string} roles - List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller. +roles: [] + +## @param {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. +resources: {} + +## @param {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). +gpus: [] + +## @param {Kubelet} [kubelet] - Kubelet resource reservations for this pool. +kubelet: {} + +## @param {string} maxUnhealthy="50%" - Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. +maxUnhealthy: "50%" + +## @param {string} nodeStartupTimeout="10m" - Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). +nodeStartupTimeout: "10m" + +## +## @section Kubernetes Version +## + +## @enum {string} Version +## @value v1.35 +## @value v1.34 +## @value v1.33 +## @value v1.32 +## @value v1.31 + +## @param {Version} version - Kubernetes major.minor version the pool joins. Must match the parent cluster's version and satisfy the Talos<->Kubernetes support matrix against `talos.version`. +version: "v1.35" + +## +## @section Talos Worker Image +## + +## @typedef {struct} Talos - Talos worker image configuration. +## @field {string} version - Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. +## @field {string} schematicID - Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. +## @field {string} imageFactoryURL - Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. +## @field {string} installerRepository - OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. + +## @param {Talos} talos - Talos worker image configuration. Keep in sync with the parent cluster's `talos`. +talos: + version: "v1.13.6" + schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" + imageFactoryURL: "https://factory.talos.dev" + installerRepository: "factory.talos.dev/installer" + +## +## @section Images +## + +## @typedef {struct} Images - Optional image overrides for chart-internal helpers. +## @field {string} [kubectl] - Image used by the talos-reconcile Job (kubectl). Empty falls back to images/kubectl.tag. + +## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries. +images: + kubectl: ""