From 92b9785daaad925888ebf270f6bf3a29cb895e18 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Thu, 16 Jul 2026 00:54:57 +0300 Subject: [PATCH] refactor(kubernetes): remove worker node pools from the chart The KubevirtMachineTemplate/MachineDeployment/MachineHealthCheck/worker WorkloadMonitor, the preserve-old-KMT block, the talos-reconcile Job and its RBAC now live in the kubernetes-nodes chart. The kubernetes chart renders the control plane only. Signed-off-by: Ivan Okhotnikov --- .../apps/kubernetes/templates/_helpers.tpl | 18 - .../apps/kubernetes/templates/_resources.tpl | 8 - .../apps/kubernetes/templates/cluster.yaml | 542 +---------------- .../templates/dashboard-resourcemap.yaml | 3 - .../templates/talos/talos-reconcile-job.yaml | 570 ------------------ 5 files changed, 9 insertions(+), 1132 deletions(-) delete mode 100644 packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index eefc6ab22..f7a2eafe5 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -135,24 +135,6 @@ enable the ingress-nginx addon, either supply an explicit nodeGroup with autoscaler will bring the default md0 up in response to the ingress-nginx controller Pods becoming Pending on install. */}} -{{- define "kubernetes.nodeGroups" -}} -{{- if .Values.nodeGroups -}} -{{ toYaml .Values.nodeGroups }} -{{- else -}} -md0: - minReplicas: 0 - maxReplicas: 10 - instanceType: "u1.medium" - diskSize: 20Gi - storageClass: "" - roles: - - ingress-nginx - resources: {} - gpus: [] - kubelet: {} -{{- end -}} -{{- end }} - {{/* OIDC clientId for the per-cluster Keycloak public client (mode: System). diff --git a/packages/apps/kubernetes/templates/_resources.tpl b/packages/apps/kubernetes/templates/_resources.tpl index fb9407616..92b4207f6 100644 --- a/packages/apps/kubernetes/templates/_resources.tpl +++ b/packages/apps/kubernetes/templates/_resources.tpl @@ -52,11 +52,3 @@ These presets are for basic testing and not meant to be used in production Convert a CPU quantity string (e.g. "100m", "0.5", "1", "4") to millicores. Returns a number string: "100", "500", "1000", "4000". */}} -{{- define "kubernetes.cpuToMillicores" -}} -{{- $str := . | toString -}} -{{- if hasSuffix "m" $str -}} -{{- trimSuffix "m" $str -}} -{{- else -}} -{{- mulf ($str | float64) 1000.0 | int | toString -}} -{{- end -}} -{{- end -}} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 35771f638..5f3ded9f2 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -36,18 +36,15 @@ flux marks the HelmRelease Ready and retries its 5m reconcile loop until the Tenant chart catches up. */}} -{{- /* The TalosConfigTemplate and the apiserver-Service ClusterIP-SAN - used to be rendered in this template behind a lookup gate that - races every fresh install: Service / Talos CA / K8s CA do not - exist on first Helm render, so the gate fails, the TCT is - skipped, and Helm's release-storage three-way merge then refuses - to upgrade because no values changed. Both objects are now - produced by the main-phase Job in - templates/talos/talos-reconcile-job.yaml, which blocks until the - four dependencies exist and then `kubectl apply`s deterministic - results. The Job owns the TCT through an ownerReference to the - KamajiControlPlane so K8s garbage-collects it naturally on - release uninstall. */}} +{{- /* Worker node pools (KubevirtMachineTemplate, MachineDeployment, + MachineHealthCheck, worker WorkloadMonitor) and the Talos + reconcile Job that produces each pool's TalosConfigTemplate now + live in the separate `kubernetes-nodes` chart (one HelmRelease + per pool). This chart renders the control plane only. The + reconcile Job in the kubernetes-nodes release still re-patches + this KamajiControlPlane's apiserver-Service ClusterIP-SAN and + owns each TalosConfigTemplate via an ownerReference to this KCP, + so garbage collection on control-plane uninstall is unchanged. */}} {{- /* Cluster pod/service CIDRs MUST match between the CAPI Cluster spec.clusterNetwork below and the Talos worker machineconfig's @@ -60,134 +57,6 @@ {{- $serviceCIDR := "10.95.0.0/16" }} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} -{{- $kubevirtmachinetemplateNames := list }} -{{- define "kubevirtmachinetemplate" -}} -spec: - {{- /* Talos has no SSH; use 'none' so CAPK does not block Machine.Ready - on an SSH probe that will never succeed. The Machine reaches Ready - once kubelet registers with the tenant apiserver via standard CSR. */}} - virtualMachineBootstrapCheck: - checkStrategy: none - virtualMachineTemplate: - metadata: - namespace: {{ $.Release.Namespace }} - labels: - {{- range .group.roles }} - node-role.kubernetes.io/{{ . }}: "" - {{- end }} - spec: - {{- /* Explicit resources take precedence over instanceType. When a node - group is sized by resources (both cpu and memory set), the - instancetype is omitted from the VM: KubeVirt rejects a - VirtualMachine that references an instancetype and also overrides - either domain.cpu or domain.memory. */}} - {{- $sizedByResources := and .group.resources .group.resources.cpu .group.resources.memory }} - {{- if and .group.instanceType (not $sizedByResources) }} - instancetype: - kind: VirtualMachineClusterInstancetype - name: {{ .group.instanceType }} - {{- end }} - runStrategy: Always - dataVolumeTemplates: - - metadata: - name: disk-system - annotations: - cdi.kubevirt.io/storage.usePopulator: "false" - spec: - {{- /* Use the openstack/configdrive Talos image: KubeVirt CAPK - injects bootstrap data via cloudInitConfigDrive (OpenStack - metadata service shape), and the openstack platform image - is the one that reads it. The nocloud-amd64 image silently - ignores configdrive and never applies the machineconfig. */}} - source: - http: - url: {{ $.Values.talos.imageFactoryURL | trimSuffix "/" }}/image/{{ $.Values.talos.schematicID }}/{{ $.Values.talos.version }}/openstack-amd64.raw.xz - {{- if hasKey .group "ephemeralStorage" }} - {{- fail (printf "nodeGroup %q: ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system." .groupName) }} - {{- end }} - storage: - resources: - requests: - storage: {{ .group.diskSize | default "20Gi" | quote }} - {{- /* - Worker VMs live-migrate, so their disks need ReadWriteMany. The - RWX access mode is supplied by the chosen StorageClass's CDI - StorageProfile — it is not set on the DataVolume here. linstor-csi - rejects RWX volumes that are not on a DRBD-backed StorageClass, so - an empty nodeGroup storageClass falls back to the application-level - storageClass (a replicated/DRBD class) instead of the cluster - default, which may be a non-DRBD class that the CSI driver would - reject. Both empty still omits the field so the cluster default - applies. - */}} - {{- with .group.storageClass | default $.Values.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} - template: - metadata: - annotations: - kubevirt.io/allow-pod-bridge-network-live-migration: "true" - labels: - apps.cozystack.io/application.group: apps.cozystack.io - apps.cozystack.io/application.kind: Kubernetes - apps.cozystack.io/application.name: {{ $.Release.Name | trimPrefix "kubernetes-" | quote }} - {{- range .group.roles }} - node-role.kubernetes.io/{{ . }}: "" - {{- end }} - cluster.x-k8s.io/deployment-name: {{ $.Release.Name }}-{{ .groupName }} - spec: - {{- if .Values._cluster.scheduling }} - {{- $rawConstraints := get .Values._cluster.scheduling "globalAppTopologySpreadConstraints" }} - {{- if $rawConstraints }} - {{- $rawConstraints | fromYaml | toYaml | nindent 10 }} - labelSelector: - matchLabels: - cluster.x-k8s.io/cluster-name: {{ $.Release.Name }} - {{- end }} - {{- end }} - domain: - {{- if and .group.resources .group.resources.cpu }} - cpu: - threads: 1 - cores: {{ .group.resources.cpu }} - sockets: 1 - {{- end }} - devices: - {{- if .group.gpus }} - gpus: - {{- range $i, $gpu := .group.gpus }} - - name: gpu{{ add $i 1 }} - deviceName: {{ $gpu.name }} - {{- end }} - {{- end }} - disks: - - name: system - disk: - bus: virtio - pciAddress: 0000:07:00.0 - blockSize: - custom: - logical: 512 - physical: 4096 - interfaces: - - name: default - bridge: {} - {{- if and .group.resources .group.resources.memory }} - memory: - guest: {{ .group.resources.memory }} - {{- end }} - resources: - {{/* 2Gi is enough: persistent DataVolumes are not counted as pod ephemeral storage */}} - {{- include "cozy-lib.resources.sanitize" (list (dict "ephemeral-storage" "2Gi") $) | nindent 14 }} - evictionStrategy: External - volumes: - - name: system - dataVolume: - name: disk-system - networks: - - name: default - pod: {} -{{- end }} {{- if not $etcd }} --- apiVersion: v1 @@ -508,397 +377,4 @@ metadata: cluster.x-k8s.io/managed-by: kamaji name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} -{{- range $groupName, $group := (include "kubernetes.nodeGroups" $ | fromYaml) }} -{{- /* instanceType and explicit resources are alternative sizing sources, and - explicit resources win. KubeVirt rejects a VirtualMachine that references - an instancetype while also overriding either domain.cpu or domain.memory, - so when a group is sized by resources (both cpu and memory set) the - instancetype is omitted from the VM (see the VM spec above) and the lookup - below is skipped — a stale instancetype name cannot fail an otherwise - valid, resource-sized group. Explicit resources are all-or-nothing: a - partial spec (only one of cpu/memory) would emit one domain override - alongside the instancetype, which KubeVirt rejects, so it fails here with - a clear render-time error instead of a late admission rejection. */}} -{{- $hasCpu := and $group.resources $group.resources.cpu }} -{{- $hasMem := and $group.resources $group.resources.memory }} -{{- if and $group.instanceType (or (and $hasCpu (not $hasMem)) (and $hasMem (not $hasCpu))) }} -{{- fail (printf "nodeGroup %s: set both resources.cpu and resources.memory, or neither" $groupName) }} -{{- end }} -{{/* Resolve instanceType once per group — reused for kubelet reservations and capacity annotations. */}} -{{- $instanceType := dict }} -{{- if and $group.instanceType (not (and $hasCpu $hasMem)) }} -{{- $instanceType = (lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType) }} -{{- if not $instanceType }} -{{- fail (printf "nodeGroup %s: specified instanceType %q not found in cluster" $groupName $group.instanceType) }} -{{- end }} -{{- end }} -{{/* Compute kubelet reservations: explicit resources.memory/cpu wins over instanceType spec. - Auto-computed system/kube reserved memory = 5% of effective memory, clamped to [256Mi, 1Gi]. - Auto-computed system/kube reserved CPU = 5% of effective CPU, clamped to [50m, 500m]. */}} -{{- $effectiveMemory := "" }} -{{- if and $group.resources $group.resources.memory }} -{{- $effectiveMemory = $group.resources.memory | toString }} -{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} -{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} -{{- end }} -{{- $effectiveCpu := "" }} -{{- if and $group.resources $group.resources.cpu }} -{{- $effectiveCpu = $group.resources.cpu | toString }} -{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} -{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} -{{- end }} -{{- $autoReservedMi := 256 }} -{{- if $effectiveMemory }} -{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} -{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} -{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} -{{- end }} -{{- $autoReservedMillicores := 50 }} -{{- if $effectiveCpu }} -{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} -{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }} -{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} -{{- end }} -{{- $kubeletOverride := $group.kubelet | default dict }} -{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} -{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} -{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} -{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} -{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} -{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }} -{{/* Validate reservation field formats. - These guards check the USER-SUPPLIED value (from values.yaml via - $kubeletOverride), not the resolved/computed default — the default is - always valid by construction. The eviction threshold validation below - differs: it validates the RESOLVED values (user-supplied or default). */}} -{{- $validations := list - (dict "key" "systemReservedMemory" "value" $kubeletOverride.systemReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)") - (dict "key" "kubeReservedMemory" "value" $kubeletOverride.kubeReservedMemory "pattern" `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` "hint" "a valid Kubernetes memory quantity (e.g. 128Mi, 1Gi, 1e9, 500e6)") - (dict "key" "systemReservedCpu" "value" $kubeletOverride.systemReservedCpu "pattern" `^([0-9]+m|[0-9]+(\.[0-9]+)?)$` "hint" "a valid Kubernetes CPU quantity (integer millicores like 100m, or decimal cores like 0.5, 1)") - (dict "key" "kubeReservedCpu" "value" $kubeletOverride.kubeReservedCpu "pattern" `^([0-9]+m|[0-9]+(\.[0-9]+)?)$` "hint" "a valid Kubernetes CPU quantity (integer millicores like 100m, or decimal cores like 0.5, 1)") -}} -{{- range $validations }} -{{- if and .value (not (regexMatch .pattern (.value | toString))) }} -{{- fail (printf "nodeGroup %s: invalid %s value %q — must be %s" $groupName .key (.value | toString) .hint) }} -{{- end }} -{{- end }} -{{/* Validate that user-supplied reservation values are positive (non-zero) */}} -{{- range $_, $entry := list (dict "key" "systemReservedMemory" "value" $kubeletOverride.systemReservedMemory) (dict "key" "kubeReservedMemory" "value" $kubeletOverride.kubeReservedMemory) }} -{{- if $entry.value }} -{{- $bytes := include "cozy-lib.resources.toFloat" ($entry.value | toString) | float64 }} -{{- if le $bytes 0.0 }} -{{- fail (printf "nodeGroup %s: %s must be a positive value, got %s" $groupName $entry.key ($entry.value | toString)) }} -{{- end }} -{{- end }} -{{- end }} -{{- range $_, $entry := list (dict "key" "systemReservedCpu" "value" $kubeletOverride.systemReservedCpu) (dict "key" "kubeReservedCpu" "value" $kubeletOverride.kubeReservedCpu) }} -{{- if $entry.value }} -{{- $cpuMillis := include "kubernetes.cpuToMillicores" $entry.value | float64 }} -{{- if le $cpuMillis 0.0 }} -{{- fail (printf "nodeGroup %s: %s must be a positive value, got %s" $groupName $entry.key ($entry.value | toString)) }} -{{- end }} -{{- end }} -{{- end }} -{{/* Validate absolute eviction threshold formats before converting */}} -{{- if not (hasSuffix "%" $evictionHardMemory) }} -{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` $evictionHardMemory) }} -{{- fail (printf "nodeGroup %s: invalid evictionHardMemory value %q — must be a percentage (e.g. 7%%) or a valid Kubernetes memory quantity (e.g. 500Mi, 1Gi, 1e9)" $groupName $evictionHardMemory) }} -{{- end }} -{{- end }} -{{- if not (hasSuffix "%" $evictionSoftMemory) }} -{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$` $evictionSoftMemory) }} -{{- fail (printf "nodeGroup %s: invalid evictionSoftMemory value %q — must be a percentage (e.g. 10%%) or a valid Kubernetes memory quantity (e.g. 500Mi, 1Gi, 1e9)" $groupName $evictionSoftMemory) }} -{{- end }} -{{- end }} -{{- if and (hasSuffix "%" $evictionHardMemory) (hasSuffix "%" $evictionSoftMemory) }} -{{- $hardPctStr := trimSuffix "%" $evictionHardMemory }} -{{- $softPctStr := trimSuffix "%" $evictionSoftMemory }} -{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $hardPctStr) }} -{{- fail (printf "nodeGroup %s: evictionHardMemory percentage value %q is not a valid number" $groupName $hardPctStr) }} -{{- end }} -{{- if not (regexMatch `^[0-9]+(\.[0-9]+)?$` $softPctStr) }} -{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage value %q is not a valid number" $groupName $softPctStr) }} -{{- end }} -{{- $hardPct := $hardPctStr | float64 }} -{{- $softPct := $softPctStr | float64 }} -{{- if le $hardPct 0.0 }} -{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $hardPctStr) }} -{{- end }} -{{- if gt $hardPct 100.0 }} -{{- fail (printf "nodeGroup %s: evictionHardMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $hardPctStr) }} -{{- end }} -{{- if le $softPct 0.0 }} -{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $softPctStr) }} -{{- end }} -{{- if gt $softPct 100.0 }} -{{- fail (printf "nodeGroup %s: evictionSoftMemory percentage must be greater than 0 and at most 100, got %s%%" $groupName $softPctStr) }} -{{- end }} -{{- if ge $hardPct $softPct }} -{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) must be strictly less than evictionSoftMemory (%s)" $groupName $evictionHardMemory $evictionSoftMemory) }} -{{- end }} -{{- else if and (not (hasSuffix "%" $evictionHardMemory)) (not (hasSuffix "%" $evictionSoftMemory)) }} -{{- $hardBytes := include "cozy-lib.resources.toFloat" $evictionHardMemory | float64 }} -{{- $softBytes := include "cozy-lib.resources.toFloat" $evictionSoftMemory | float64 }} -{{- if ge $hardBytes $softBytes }} -{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) must be strictly less than evictionSoftMemory (%s)" $groupName $evictionHardMemory $evictionSoftMemory) }} -{{- end }} -{{- else }} -{{- fail (printf "nodeGroup %s: evictionHardMemory (%s) and evictionSoftMemory (%s) must use the same unit type (both percentage or both absolute)" $groupName $evictionHardMemory $evictionSoftMemory) }} -{{- end }} -{{/* Resolve eviction hard threshold to bytes for arithmetic. - Percentage is relative to effective node memory; absolute values are used as-is. */}} -{{- $evictionHardBytes := 0.0 }} -{{- if $effectiveMemory }} -{{- if hasSuffix "%" $evictionHardMemory }} -{{- $evictionPct := trimSuffix "%" $evictionHardMemory | float64 }} -{{- $evictionHardBytes = mulf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) (divf $evictionPct 100.0) }} -{{- else }} -{{- $evictionHardBytes = include "cozy-lib.resources.toFloat" $evictionHardMemory | float64 }} -{{- end }} -{{- end }} -{{/* Validate that total reserved memory (system + kube + eviction) does not exceed effective node memory */}} -{{- if $effectiveMemory }} -{{- $sysBytes := include "cozy-lib.resources.toFloat" $systemReservedMemory | float64 }} -{{- $kubeBytes := include "cozy-lib.resources.toFloat" $kubeReservedMemory | float64 }} -{{- $totalReservedBytes := addf (addf $sysBytes $kubeBytes) $evictionHardBytes }} -{{- $effectiveMemBytes := include "cozy-lib.resources.toFloat" $effectiveMemory | float64 }} -{{- if ge $totalReservedBytes $effectiveMemBytes }} -{{- fail (printf "nodeGroup %s: total reserved memory (systemReservedMemory=%s + kubeReservedMemory=%s + evictionHardMemory=%s) must be less than effective node memory (%s)" $groupName $systemReservedMemory $kubeReservedMemory $evictionHardMemory $effectiveMemory) }} -{{- end }} -{{- end }} -{{/* Validate that total reserved CPU does not exceed effective node CPU */}} -{{- if $effectiveCpu }} -{{- $sysCpuMillis := include "kubernetes.cpuToMillicores" $systemReservedCpu | float64 }} -{{- $kubeCpuMillis := include "kubernetes.cpuToMillicores" $kubeReservedCpu | float64 }} -{{- $totalCpuMillis := addf $sysCpuMillis $kubeCpuMillis }} -{{- $effectiveCpuMillicoresV := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} -{{- $effectiveCpuMillis := float64 $effectiveCpuMillicoresV }} -{{- if ge $totalCpuMillis $effectiveCpuMillis }} -{{- fail (printf "nodeGroup %s: total reserved CPU (systemReservedCpu=%s + kubeReservedCpu=%s) must be less than effective node CPU (%s cores)" $groupName $systemReservedCpu $kubeReservedCpu $effectiveCpu) }} -{{- end }} -{{- end }} -{{/* Guard: explicit kubelet overrides require effective memory for validation */}} -{{- if not $effectiveMemory }} -{{- $hasOverride := or $kubeletOverride.systemReservedMemory $kubeletOverride.kubeReservedMemory $kubeletOverride.evictionHardMemory $kubeletOverride.evictionSoftMemory }} -{{- if $hasOverride }} -{{- fail (printf "nodeGroup %s: kubelet memory reservations require effective memory to validate — set resources.memory or instanceType" $groupName) }} -{{- end }} -{{- end }} -{{/* Guard: explicit kubelet CPU overrides require effective CPU for validation */}} -{{- if not $effectiveCpu }} -{{- $hasCpuOverride := or $kubeletOverride.systemReservedCpu $kubeletOverride.kubeReservedCpu }} -{{- if $hasCpuOverride }} -{{- fail (printf "nodeGroup %s: cannot validate kubelet CPU reservations without effective CPU — set resources.cpu or a valid instanceType" $groupName) }} -{{- end }} -{{- end }} ---- -{{- /* TalosConfigTemplate is no longer rendered here — the chart cannot - produce a valid Talos worker machineconfig at Helm template time - because three of the inputs (apiserver Service ClusterIP, Talos - CA from cert-manager, Kamaji-issued tenant CA) are created - asynchronously after Helm applies this manifest. The main-phase - Job in templates/talos/talos-reconcile-job.yaml waits for those - inputs and produces the TalosConfigTemplate with a - KamajiControlPlane ownerReference for natural GC. That Job runs in - the main install phase, so it applies the template as soon as the - inputs appear — in parallel with the rest of the install. The - MachineDeployment below still references the TCT name; CAPI's - Machine controller blocks on its absence and unblocks the moment - the Job applies it. */}} ---- -{{- $context := deepCopy $ }} -{{- $_ := set $context "group" $group }} -{{- $_ := set $context "groupName" $groupName }} -{{- $kubevirtmachinetemplate := include "kubevirtmachinetemplate" $context }} -{{- $kubevirtmachinetemplateHash := $kubevirtmachinetemplate | sha256sum | trunc 6 }} -{{- $kubevirtmachinetemplateName := printf "%s-%s-%s" $.Release.Name $groupName $kubevirtmachinetemplateHash }} -{{- $kubevirtmachinetemplateNames = append $kubevirtmachinetemplateNames $kubevirtmachinetemplateName }} -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -kind: KubevirtMachineTemplate -metadata: - name: {{ $.Release.Name }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }} - namespace: {{ $.Release.Namespace }} -spec: - template: - {{- $kubevirtmachinetemplate | nindent 4 }} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: {{ $.Release.Name }}-{{ $groupName }} - namespace: {{ $.Release.Namespace }} - annotations: - cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "{{ $group.minReplicas }}" - cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "{{ $group.maxReplicas }}" - {{- if $effectiveMemory }} - {{- $sysBytes := include "cozy-lib.resources.toFloat" $systemReservedMemory | float64 }} - {{- $kubeBytes := include "cozy-lib.resources.toFloat" $kubeReservedMemory | float64 }} - {{- $totalReservedBytes := addf (addf $sysBytes $kubeBytes) $evictionHardBytes }} - {{- $allocatableBytes := subf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) $totalReservedBytes }} - {{- $allocatableMi := divf $allocatableBytes 1048576.0 | floor }} - capacity.cluster-autoscaler.kubernetes.io/memory: "{{ $allocatableMi }}Mi" - {{- end }} - {{- if $effectiveCpu }} - {{- $sysCpuMillisA := include "kubernetes.cpuToMillicores" $systemReservedCpu | float64 }} - {{- $kubeCpuMillisA := include "kubernetes.cpuToMillicores" $kubeReservedCpu | float64 }} - {{- $effectiveCpuMillicoresA := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} - {{- $effectiveCpuMillisA := float64 $effectiveCpuMillicoresA }} - {{- $allocatableCpuMillis := subf $effectiveCpuMillisA (addf $sysCpuMillisA $kubeCpuMillisA) }} - {{- $allocatableCpuMillisInt := $allocatableCpuMillis | ceil | int }} - capacity.cluster-autoscaler.kubernetes.io/cpu: "{{ $allocatableCpuMillisInt }}m" - {{- end }} -spec: - clusterName: {{ $.Release.Name }} - # `spec.replicas` is intentionally omitted so the field is owned by - # the cluster-autoscaler alone. Two independent CAPI mechanisms — - # not Server-Side-Apply field managers — carry the guarantee: - # 1. Create: the MachineDeployment defaulting webhook seeds - # `spec.replicas` from the `cluster-api-autoscaler-node-group- - # min-size` annotation (see - # https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling). - # 2. Update: `calculateMachineDeploymentReplicas` preserves the - # existing live value (`oldMD.Spec.Replicas`) whenever the - # incoming spec has `replicas: nil`, clamping into - # `[min, max]` if it drifts outside — so a cluster the - # autoscaler holds above min-size is not drained by a - # `helm upgrade` that would otherwise re-render this field. - # Flux helm-controller applies this chart through the Helm SDK - # three-way merge (not SSA), so field-manager ownership is not the - # mechanism at play. The chart guarantee reduces to "leave the - # field out of the rendered manifest and let CAPI's update-time - # defaulting keep the live value stable". Emitting a chart-managed - # `replicas: N` here (as this chart did before - # cozystack/cozystack#3231) drains workers on every `helm upgrade` - # action until the autoscaler notices unschedulable Pods and - # scales back up — a real disruption window on every routine - # platform bump. - strategy: - rollingUpdate: - maxSurge: {{ $group.maxReplicas }} - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: {{ $.Release.Name }} - cluster.x-k8s.io/deployment-name: {{ $.Release.Name }}-{{ $groupName }} - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: {{ $.Release.Name }} - cluster.x-k8s.io/deployment-name: {{ $.Release.Name }}-{{ $groupName }} - {{- range $group.roles }} - node-role.kubernetes.io/{{ . }}: "" - {{- end }} - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: TalosConfigTemplate - name: {{ $.Release.Name }}-{{ $groupName }} - namespace: {{ $.Release.Namespace }} - clusterName: {{ $.Release.Name }} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 - kind: KubevirtMachineTemplate - name: {{ $.Release.Name }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }} - namespace: {{ $.Release.Namespace }} - version: {{ include "kubernetes.versionMap" $}} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: {{ $.Release.Name }}-{{ $groupName }} - namespace: {{ $.Release.Namespace }} -spec: - clusterName: {{ $.Release.Name }} - {{- /* MHC spec.maxUnhealthy is an IntOrString. CAPI admission accepts a - bare integer (0, 1, ...) or a percentage string ("0%", "50%"), but - rejects a quoted plain integer ("0"). Quote only when the operator - supplied a percentage; otherwise render the value as a bare int so - the admission webhook is happy with the chart-default "0". A node - group may override the cluster-wide nodeHealthCheck via - nodeGroups..maxUnhealthy / .nodeStartupTimeout. */}} - {{- $maxUnhealthy := toString $.Values.nodeHealthCheck.maxUnhealthy }} - {{- $muSource := "nodeHealthCheck.maxUnhealthy" }} - {{- if hasKey $group "maxUnhealthy" }} - {{- $maxUnhealthy = toString $group.maxUnhealthy }} - {{- $muSource = printf "nodeGroups.%s.maxUnhealthy" $groupName }} - {{- end }} - {{- if hasSuffix "%" $maxUnhealthy }} - {{- if not (regexMatch "^[0-9]+%$" $maxUnhealthy) }} - {{- fail (printf "%s must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $muSource $maxUnhealthy) }} - {{- end }} - maxUnhealthy: {{ $maxUnhealthy | quote }} - {{- else if regexMatch "^[0-9]+$" $maxUnhealthy }} - maxUnhealthy: {{ $maxUnhealthy | int }} - {{- else }} - {{- fail (printf "%s must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $muSource $maxUnhealthy) }} - {{- end }} - {{- $nodeStartupTimeout := $.Values.nodeHealthCheck.nodeStartupTimeout }} - {{- if hasKey $group "nodeStartupTimeout" }}{{ $nodeStartupTimeout = $group.nodeStartupTimeout }}{{ end }} - nodeStartupTimeout: {{ $nodeStartupTimeout }} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: {{ $.Release.Name }} - cluster.x-k8s.io/deployment-name: {{ $.Release.Name }}-{{ $groupName }} - unhealthyConditions: - - type: Ready - status: Unknown - timeout: 30s - - type: Ready - status: "False" - timeout: 300s ---- -apiVersion: cozystack.io/v1alpha1 -kind: WorkloadMonitor -metadata: - name: {{ $.Release.Name }}-{{ $groupName }} - namespace: {{ $.Release.Namespace }} -spec: - minReplicas: {{ $group.minReplicas }} - kind: kubernetes - type: worker - selector: - cluster.x-k8s.io/cluster-name: {{ $.Release.Name }} - cluster.x-k8s.io/deployment-name: {{ $.Release.Name }}-{{ $groupName }} - cluster.x-k8s.io/role: worker - version: {{ $.Chart.Version }} -{{- end }} ---- -{{- /* -We must preserve all previous KubevirtMachineTemplates until a MachineSet references them. -*/ -}} -{{- $mss := (lookup "cluster.x-k8s.io/v1beta1" "MachineSet" $.Release.Namespace "").items }} -{{- $oldKubevirtmachinetemplates := dict }} -{{- range $kmt := (lookup "infrastructure.cluster.x-k8s.io/v1alpha1" "KubevirtMachineTemplate" .Release.Namespace "").items }} -{{- range $or := $kmt.metadata.ownerReferences }} -{{- /* Only re-emit KubevirtMachineTemplates that belong to THIS release. The - Cluster ownerReference alone is not enough: worker pools split out into - kubernetes-nodes releases share the same CAPI Cluster ownerReference, so - without the release-name scoping this block would re-emit a pool's KMT - (annotated meta.helm.sh/release-name=kubernetes-nodes--) - and the next upgrade of this release would fail Helm ownership - validation. */}} -{{- if and (eq $or.kind "Cluster") (eq $or.name $.Release.Name) (eq (dig "annotations" "meta.helm.sh/release-name" "" $kmt.metadata) $.Release.Name) }} -{{- range $ms := $mss }} -{{- if and (eq $ms.spec.template.spec.infrastructureRef.kind "KubevirtMachineTemplate") (eq $ms.spec.template.spec.infrastructureRef.name $kmt.metadata.name) }} -{{- if not (has $kmt.metadata.name $kubevirtmachinetemplateNames) }} -{{- $oldKubevirtmachinetemplates = merge $oldKubevirtmachinetemplates (dict $kmt.metadata.name $kmt) }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} - -{{- range $oldKubevirtmachinetemplates }} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -kind: KubevirtMachineTemplate -metadata: - name: {{ .metadata.name }} - namespace: {{ .metadata.Namespace }} -spec: - {{- .spec | toYaml | nindent 2 }} -{{- end }} {{- end }} diff --git a/packages/apps/kubernetes/templates/dashboard-resourcemap.yaml b/packages/apps/kubernetes/templates/dashboard-resourcemap.yaml index 17590c688..8769365d1 100644 --- a/packages/apps/kubernetes/templates/dashboard-resourcemap.yaml +++ b/packages/apps/kubernetes/templates/dashboard-resourcemap.yaml @@ -30,9 +30,6 @@ rules: - workloadmonitors resourceNames: - {{ .Release.Name }} - {{- range $groupName, $group := (include "kubernetes.nodeGroups" $ | fromYaml) }} - - {{ $.Release.Name }}-{{ $groupName }} - {{- end }} verbs: ["get", "list", "watch"] --- kind: RoleBinding diff --git a/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml b/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml deleted file mode 100644 index 9e2ed3144..000000000 --- a/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml +++ /dev/null @@ -1,570 +0,0 @@ -{{- /* - Main-phase Job (NOT a Helm hook) that resolves the chart's - lookup-and-reuse race for TalosConfigTemplate and - KamajiControlPlane.spec.network.certSANs. - - Background. The Talos worker machineconfig and the apiserver Service - ClusterIP-SAN both need values that only exist AFTER Helm has applied - the first render: the Service is emitted by the Kamaji controller in - reaction to the KamajiControlPlane CR, the Talos-CA TLS Secret is - issued by cert-manager from the Talos PKI Certificate CR, and the - Kubernetes-CA Secret is generated by Kamaji as part of its tenant - control-plane bootstrap. A Helm template that uses `lookup` to read - any of these into the rendered manifest fails the race on every fresh - install — the lookup returns nil, the gated TalosConfigTemplate is - skipped, and Helm's release-storage three-way merge concludes the - release is in-sync on every subsequent reconcile because no chart - value changed. The result is a tenant whose KamajiControlPlane is - Ready but whose CAPI MachineDeployment is stuck on - `cannot create a new MachineSet when templates do not exist` forever. - - This Job blocks until the four dependencies are observably present, - then produces both objects via `kubectl apply` (idempotent). The - TalosConfigTemplate carries an ownerReference to the KamajiControlPlane - so Kubernetes garbage-collects it through the standard owner-reference - cascade when the chart's release is uninstalled — Helm itself does not - need to track the resource, and a `pre-delete` hook is not required. - The certSANs patch is a merge against the KCP `spec.network.certSANs` - field; the Kamaji controller picks up the change and re-issues the - apiserver TLS cert with the live ClusterIP in its SAN list. - - Why main phase, not a post-install hook. This Job is the only thing - that creates the TalosConfigTemplate (the chart no longer renders it — - see cluster.yaml). As a post-install hook it runs only AFTER Helm has - applied the release's main resources and, when Helm waits, after they - become Ready. But the same release also emits in-tenant addon - HelmReleases — notably cilium — whose readiness needs tenant worker - nodes, and those nodes can only be created once this Job has produced - the TalosConfigTemplate the worker MachineDeployment clones from. So - when the main-resource wait is in effect it DEADLOCKS: the install - waits for cilium to be Ready, cilium waits for worker nodes, the nodes - wait for the TalosConfigTemplate, and the template waits for this hook - — which never runs, because the install never finishes the wait that - precedes the post-install hook phase. The install times out and the - MachineDeployment never scales. (Observed on a failing run: the parent - HelmRelease stuck at `Running install action 20m`, NEITHER post-install - hook ever created, the cilium sub-release timing out on cilium-operator - — all while the four inputs were ready early. The hook never reached; - the inputs were not late.) - - The release sets `helm-install-disable-wait` precisely to skip that - main-resource wait so the hooks run, but its effect is unreliable — - which is what makes the failure bimodal: when the wait is skipped the - hook runs and the template lands in ~46s; when it is not, the deadlock - above strands the install. Rendering this Job as an ordinary MAIN-phase - resource removes the dependency on both the hook phase and DisableWait. - The Job is created during the apply itself (not after the wait) and - runs on the management cluster — which has nodes, unlike the tenant - cilium-operator — so it produces the TalosConfigTemplate, the - MachineDeployment scales, workers join, cilium goes Ready, and the - main-resource wait (if any) closes within the install budget: the Job - completes in tens of seconds, node boot plus cilium readiness is a few - minutes. It works whether or not DisableWait takes effect. The Job's - RBAC, ServiceAccount and egress CiliumNetworkPolicy are main-phase too; - the wait loop tolerates its dependencies (and its own network policy) - not yet being in place, so Helm's kind-ordered apply needs no explicit - sequencing. The bootstrap-token Job stays a post-install hook. - - Churn-free across upgrades. The Job is immutable (spec.template cannot - be patched), so it carries a content-hash name suffix — mirrors the - KubevirtMachineTemplate idiom in cluster.yaml. An identical render - yields the same name and Helm no-ops a Job it can never patch; any - change to a field that flows into the rendered Job spec (and therefore - into the TalosConfigTemplate it applies) yields a new name, so a fresh - Job re-applies the updated template and Helm prunes the superseded one. - Because the hash is taken over the whole rendered spec, no spec change - can ever land on the same name and trigger an immutable-field patch - error. The applied TalosConfigTemplate keeps a stable name - (-, referenced by MachineDeployment.bootstrap.configRef) - regardless of the Job's hash, so an unchanged render re-applies a - byte-identical TCT — a CAPI no-op that never rolls existing workers. -*/}} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-talos-reconcile - namespace: {{ .Release.Namespace }} - labels: - {{- include "kubernetes.labels" . | nindent 4 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-talos-reconcile - namespace: {{ .Release.Namespace }} - labels: - {{- include "kubernetes.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["services"] - resourceNames: ["{{ .Release.Name }}"] - verbs: ["get"] - - apiGroups: [""] - resources: ["secrets"] - resourceNames: - - "{{ .Release.Name }}-talos-secrets" - - "{{ .Release.Name }}-talos-ca" - - "{{ .Release.Name }}-ca" - verbs: ["get"] - - apiGroups: ["controlplane.cluster.x-k8s.io"] - resources: ["kamajicontrolplanes"] - resourceNames: ["{{ .Release.Name }}"] - verbs: ["get", "patch"] - - apiGroups: ["bootstrap.cluster.x-k8s.io"] - resources: ["talosconfigtemplates"] - verbs: ["get", "create", "patch", "update"] ---- -{{- /* The tenant-root chart installs a default-deny egress policy on every - pod in the tenant namespace; without an explicit allow, this Job - cannot reach the management-cluster apiserver to read the Service - ClusterIP / Secrets / KamajiControlPlane it has to reconcile. Cilium - policies are additive, so this rule layers on top of the tenant - default-deny without weakening it for anything else. Endpoint - selector is scoped to pods carrying the reconcile label only. */}} -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: {{ .Release.Name }}-talos-reconcile - namespace: {{ .Release.Namespace }} - labels: - {{- include "kubernetes.labels" . | nindent 4 }} -spec: - endpointSelector: - matchLabels: - cozystack.io/talos-reconcile: "{{ .Release.Name }}" - egress: - - toEntities: - - kube-apiserver - - toEndpoints: - - matchLabels: - "k8s:io.kubernetes.pod.namespace": kube-system - "k8s:k8s-app": kube-dns - toPorts: - - ports: - - port: "53" - protocol: UDP - - port: "53" - protocol: TCP ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-talos-reconcile - namespace: {{ .Release.Namespace }} - labels: - {{- include "kubernetes.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-talos-reconcile -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-talos-reconcile - namespace: {{ .Release.Namespace }} ---- -{{- /* Cluster-scoped read on the management CoreDNS Service so the Job - can look up its ClusterIP at runtime and inject it into the worker - machineconfig as a host-level nameserver. Tenant workers run as - KubeVirt VMs inside the management pod network; tenant in-cluster - pods still use kubelet --cluster-dns (tenant CoreDNS) via - dnsPolicy: ClusterFirst, but the worker host itself (kubelet's - mount syscall path, image pulls) consults /etc/resolv.conf which - Talos populates from machine.network.nameservers. Without - management CoreDNS as host resolver, mounting NFS from - kubevirt-csi (linstor-csi-nfs.cozy-linstor.svc) fails with - NXDOMAIN. - - Ubuntu+kubeadm workers in main inherit pod DNS via cloud-init - and got management CoreDNS for free; Talos reads only what is - in machineconfig, so the chart has to plumb it explicitly. - - resourceNames pins the grant to the single well-known Service - name kube-dns — cluster-wide get on services scoped to that - single name is functionally a single-resource permission. */}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns - labels: - {{- include "kubernetes.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["services"] - resourceNames: ["kube-dns"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns - labels: - {{- include "kubernetes.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-talos-reconcile - namespace: {{ .Release.Namespace }} -{{- /* Job spec rendered through a named template so the Job name can carry - a content hash of the whole spec (see the header comment). The dot - passed in is a deepCopy of the root context with the per-nodeGroup - computed values set on it, so $.Release / $.Values resolve as usual - and the loop/computed locals are read as .group, .groupName, etc. */}} -{{- define "kubernetes.talos-reconcile-job-spec" -}} -spec: - {{- /* Each pod attempt runs the ~10-minute (120 × 5s) input-wait loop, so this - backoffLimit sets the self-heal window to roughly 5 hours. The Job does - not block the install — when disable-wait is in effect Helm does not wait - on it, and when Helm does wait the Job completes in tens of seconds — so a - long retry budget costs nothing on the install path. It only governs how - long a Job whose inputs are unusually slow keeps retrying on its own - before giving up. Bounded (not infinite) so a permanently-broken input - eventually stops spawning pods. */}} - backoffLimit: 30 - {{- /* ttlSecondsAfterFinished clears the Job (Complete or Failed) shortly after - it finishes, so a later reconcile can recreate and retry it. Without the - TTL a Failed Job would keep the same content-hash name and be a no-op on - every later upgrade — Helm never patches the immutable Job, so it would - stay Failed until deleted by hand. No churn from the TTL: the repo runs no - Flux drift detection, so an interval reconcile never re-applies a cleared - Job; only an actual upgrade does, and that re-applies the byte-identical - TalosConfigTemplate (a CAPI no-op). Mirrors the original hook Job's value. */}} - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - cozystack.io/talos-reconcile: {{ $.Release.Name | quote }} - spec: - restartPolicy: OnFailure - serviceAccountName: {{ $.Release.Name }}-talos-reconcile - containers: - - name: kubectl - image: "{{ .kubectlImage }}" - env: - - name: RELEASE - value: {{ $.Release.Name | quote }} - - name: NS - value: {{ $.Release.Namespace | quote }} - - name: GROUP_NAME - value: {{ .groupName | quote }} - - name: TALOS_TOKEN - valueFrom: - secretKeyRef: - name: {{ $.Release.Name }}-talos-secrets - key: token - - name: CLUSTER_ID - valueFrom: - secretKeyRef: - name: {{ $.Release.Name }}-talos-secrets - key: clusterId - - name: CLUSTER_SECRET - valueFrom: - secretKeyRef: - name: {{ $.Release.Name }}-talos-secrets - key: clusterSecret - - name: BOOTSTRAP_TOKEN - valueFrom: - secretKeyRef: - name: {{ $.Release.Name }}-talos-secrets - key: bootstrapToken - command: - - sh - - -ec - - | - echo "[$(date -u +%H:%M:%S)] talos-reconcile/${GROUP_NAME}: waiting for Service ClusterIP, Talos CA, K8s CA, management CoreDNS" - for i in $(seq 1 120); do - SVC_IP=$(kubectl -n "$NS" get svc "$RELEASE" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true) - TALOS_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-talos-ca" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) - K8S_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true) - COREDNS_IP=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true) - if [ -n "$SVC_IP" ] && [ -n "$TALOS_CA_B64" ] && [ -n "$K8S_CA_B64" ] && [ -n "$COREDNS_IP" ]; then - break - fi - sleep 5 - done - if [ -z "$SVC_IP" ] || [ -z "$TALOS_CA_B64" ] || [ -z "$K8S_CA_B64" ] || [ -z "$COREDNS_IP" ]; then - echo "[$(date -u +%H:%M:%S)] timed out waiting for lookup inputs (svc='$SVC_IP' talos-ca=$([ -n "$TALOS_CA_B64" ] && echo ok) k8s-ca=$([ -n "$K8S_CA_B64" ] && echo ok) coredns='$COREDNS_IP')" - exit 1 - fi - echo "[$(date -u +%H:%M:%S)] inputs ready (svc=${SVC_IP} coredns=${COREDNS_IP})" - - # Patch KamajiControlPlane.spec.network.certSANs so the apiserver - # TLS cert covers the live Service ClusterIP that Talos workers - # dial. JSON-merge-patch is array-replace semantics, which is what - # we want here — Kamaji owns the full SAN list. - kubectl -n "$NS" patch kamajicontrolplane "$RELEASE" --type=merge --patch "$( - cat <..svc") - the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The - kubevirt-csi NFS mount path is the load-bearing case: kubelet on - the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc` - (no domain suffix), and without searchDomains in /etc/resolv.conf - the host resolver hands that name to management CoreDNS verbatim - and gets NXDOMAIN. - - Distinct from tenantClusterDomain ("cluster.local"), which is the - *tenant* kubelet --cluster-dns zone. This value is the - *management* cluster-domain, sourced from .Values._cluster - (populated by cozystack-controller from the platform config). */}} -{{- $mgmtClusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- range $groupName, $group := .Values.nodeGroups }} -{{- /* Mirror the kubelet-reservation computation from cluster.yaml so the - worker machineconfig the Job applies carries the same numbers the - MachineDeployment validator was happy with. Auto-computed - system/kube reserved memory = 5% of effective memory clamped to - [256Mi, 1Gi]; auto-computed system/kube reserved CPU = 5% of - effective CPU clamped to [50m, 500m]. instanceType lookup is - optional — operators that override resources.cpu / resources.memory - on the nodeGroup get those, otherwise the chart falls back to the - same static-defaults pair the cluster.yaml validator uses on a - no-instanceType setup. */}} -{{- $instanceType := dict }} -{{- if $group.instanceType }} -{{- $instanceType = lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType }} -{{- $instanceType = $instanceType | default dict }} -{{- end }} -{{- $effectiveMemory := "" }} -{{- if and $group.resources $group.resources.memory }} -{{- $effectiveMemory = $group.resources.memory | toString }} -{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} -{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} -{{- end }} -{{- $effectiveCpu := "" }} -{{- if and $group.resources $group.resources.cpu }} -{{- $effectiveCpu = $group.resources.cpu | toString }} -{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} -{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} -{{- end }} -{{- $autoReservedMi := 256 }} -{{- if $effectiveMemory }} -{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} -{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} -{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} -{{- end }} -{{- $autoReservedMillicores := 50 }} -{{- if $effectiveCpu }} -{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} -{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }} -{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} -{{- end }} -{{- $kubeletOverride := $group.kubelet | default dict }} -{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} -{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} -{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} -{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} -{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} -{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }} -{{- /* Render the Job spec, then hash it to a name suffix so the immutable - Job is churn-free across upgrades (see the header comment). Mirrors - the KubevirtMachineTemplate content-hash idiom in cluster.yaml. */}} -{{- $jobContext := deepCopy $ }} -{{- $_ := set $jobContext "group" $group }} -{{- $_ := set $jobContext "groupName" $groupName }} -{{- $_ := set $jobContext "kubeletVersion" $kubeletVersion }} -{{- $_ := set $jobContext "talosVersion" $talosVersion }} -{{- $_ := set $jobContext "podCIDR" $podCIDR }} -{{- $_ := set $jobContext "serviceCIDR" $serviceCIDR }} -{{- $_ := set $jobContext "dnsDomain" $dnsDomain }} -{{- $_ := set $jobContext "mgmtClusterDomain" $mgmtClusterDomain }} -{{- $_ := set $jobContext "systemReservedCpu" $systemReservedCpu }} -{{- $_ := set $jobContext "systemReservedMemory" $systemReservedMemory }} -{{- $_ := set $jobContext "kubeReservedCpu" $kubeReservedCpu }} -{{- $_ := set $jobContext "kubeReservedMemory" $kubeReservedMemory }} -{{- $_ := set $jobContext "evictionHardMemory" $evictionHardMemory }} -{{- $_ := set $jobContext "evictionSoftMemory" $evictionSoftMemory }} -{{- $_ := set $jobContext "kubectlImage" (default ($.Files.Get "images/kubectl.tag" | trim) $.Values.images.kubectl) }} -{{- $jobSpec := include "kubernetes.talos-reconcile-job-spec" $jobContext }} -{{- $jobHash := $jobSpec | sha256sum | trunc 6 }} -{{- /* Truncate the name prefix, never the hash, so the name stays within the - 63-char DNS-label limit for long release/group names while the 6-char - content hash that keeps the immutable Job churn-free is always preserved. - A no-op for normal-length names (prefix << 56). */}} -{{- $jobName := printf "%s-talos-reconcile-%s" $.Release.Name $groupName | trunc 56 | trimSuffix "-" }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $jobName }}-{{ $jobHash }} - namespace: {{ $.Release.Namespace }} - labels: - {{- include "kubernetes.labels" $ | nindent 4 }} -{{ $jobSpec }} -{{- end }}