diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 57153057d..02e48cbe8 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -40,6 +40,12 @@ type ConfigSpec struct { // Optional image overrides for air-gapped or rate-limited registries. // +kubebuilder:default:={} Images Images `json:"images"` + // Talos worker image configuration. + // +kubebuilder:default:={} + Talos Talos `json:"talos"` + // MachineHealthCheck tuning for worker node groups. + // +kubebuilder:default:={} + NodeHealthCheck NodeHealthCheck `json:"nodeHealthCheck"` } type APIServer struct { @@ -177,6 +183,12 @@ type HAMiAddon struct { } type Images struct { + // Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag. + // +kubebuilder:default:="" + Kubectl string `json:"kubectl,omitempty"` + // Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag. + // +kubebuilder:default:="" + TalosCsrSigner string `json:"talosCsrSigner,omitempty"` // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. // +kubebuilder:default:="" WaitForKubeconfig string `json:"waitForKubeconfig,omitempty"` @@ -239,7 +251,7 @@ type MonitoringAgentsAddon struct { } type NodeGroup struct { - // Persistent disk size for kubelet and containerd data. + // System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact. // +kubebuilder:default:="20Gi" DiskSize resource.Quantity `json:"diskSize"` // List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). @@ -263,6 +275,15 @@ type NodeGroup struct { StorageClass string `json:"storageClass,omitempty"` } +type NodeHealthCheck struct { + // Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers. + // +kubebuilder:default:="50%" + MaxUnhealthy string `json:"maxUnhealthy"` + // Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). + // +kubebuilder:default:="10m" + NodeStartupTimeout string `json:"nodeStartupTimeout"` +} + type OuroborosAddon struct { // Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride. // +kubebuilder:default:=false @@ -288,6 +309,15 @@ type Scheduler struct { ResourcesPreset ResourcesPreset `json:"resourcesPreset"` } +type Talos struct { + // Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. + // +kubebuilder:default:="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" + SchematicID string `json:"schematicID"` + // Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. + // +kubebuilder:default:="v1.13.0" + Version string `json:"version"` +} + type VeleroAddon struct { // Enable Velero. // +kubebuilder:default:=false @@ -309,5 +339,5 @@ type IngressNginxExposeMethod string // +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge" type ResourcesPreset string -// +kubebuilder:validation:Enum="v1.35";"v1.34";"v1.33";"v1.32";"v1.31";"v1.30" +// +kubebuilder:validation:Enum="v1.35";"v1.34";"v1.33";"v1.32";"v1.31" type Version string diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 8dc8271b3..964301751 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -138,6 +138,8 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) out.Images = in.Images + out.Talos = in.Talos + out.NodeHealthCheck = in.NodeHealthCheck } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -406,6 +408,21 @@ func (in *NodeGroup) DeepCopy() *NodeGroup { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeHealthCheck) DeepCopyInto(out *NodeHealthCheck) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeHealthCheck. +func (in *NodeHealthCheck) DeepCopy() *NodeHealthCheck { + if in == nil { + return nil + } + out := new(NodeHealthCheck) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OuroborosAddon) DeepCopyInto(out *OuroborosAddon) { *out = *in @@ -455,6 +472,21 @@ func (in *Scheduler) DeepCopy() *Scheduler { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Talos) DeepCopyInto(out *Talos) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Talos. +func (in *Talos) DeepCopy() *Talos { + if in == nil { + return nil + } + out := new(Talos) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VeleroAddon) DeepCopyInto(out *VeleroAddon) { *out = *in diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index c2f44309b..e3a70869d 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -8,7 +8,6 @@ include ../../../hack/package.mk test: helm unittest . - bash tests/containerd-config-path-test.sh generate: cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go @@ -18,21 +17,17 @@ update: hack/update-versions.sh make generate -image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler +image: image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-talos-csr-signer -image-ubuntu-container-disk: - $(foreach ver,$(KUBERNETES_VERSIONS), \ - docker buildx build images/ubuntu-container-disk \ - --build-arg KUBERNETES_VERSION=$(ver) \ - --tag $(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG) \ - $(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$(ver)) \ - $(call cache-args,ubuntu-container-disk,$(ver)-buildcache) \ - --metadata-file images/ubuntu-container-disk-$(ver).json \ - $(BUILDX_ARGS) && \ - echo "$(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$(ver).json -o json -r)" \ - > images/ubuntu-container-disk-$(ver).tag && \ - rm -f images/ubuntu-container-disk-$(ver).json; \ - ) +image-talos-csr-signer: + docker buildx build images/talos-csr-signer \ + $(call image-tags,talos-csr-signer,$(KUBERNETES_PKG_TAG)) \ + $(call cache-args,talos-csr-signer) \ + --metadata-file images/talos-csr-signer.json \ + $(BUILDX_ARGS) + echo "$(REGISTRY)/talos-csr-signer:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/talos-csr-signer.json -o json -r)" \ + > images/talos-csr-signer.tag + rm -f images/talos-csr-signer.json image-kubevirt-cloud-provider: docker buildx build images/kubevirt-cloud-provider \ diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 1912419f0..93c3472c4 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -85,7 +85,9 @@ See the reference for components utilized in this service: ## Breaking Changes -- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize` to better reflect its purpose (persistent disk for kubelet and containerd data). Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs will be automatically rolling-updated via CAPI when the new values are applied. State persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh PVC. +- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize`. Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs are rolling-updated via CAPI when the new values are applied. With the Talos worker rollover in this release the field now sizes the **single system disk** (Talos OS image streamed from `factory.talos.dev`, kubelet state, containerd image cache, local-path PVCs) — the pre-Talos `disk-kubelet` PVC layout was removed. State on that single disk persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh disk. + +- **Air-gapped tenant workers temporarily unsupported (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos image fetched over HTTPS from `factory.talos.dev` and a `factory.talos.dev/installer/...` installer reference, neither of which is currently overridable. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig and was removed in this release. Tenants behind a strict-egress firewall or relying on `registries.mirrors` for tenant workers will be unable to create new workers until Phase 2 restores this via `machine.registries.mirrors` knobs and an installable/imageBase override. Tracked as a follow-up to this Phase 1 PR — file an issue if you depend on this and it is not yet landed. > The top-level `storageClass` field is annotated as immutable in the chart schema — see [`docs/storage-immutability.md`](../../../docs/storage-immutability.md) for the contract and which consumers enforce it. The per-node-group `nodeGroups[name].storageClass` field is intentionally **not** annotated immutable: it is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. @@ -106,7 +108,7 @@ See the reference for components utilized in this service: | `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` | | `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` | | `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` | -| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` | +| `nodeGroups[name].diskSize` | System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact. | `quantity` | `20Gi` | | `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` | | `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` | | `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` | @@ -168,33 +170,53 @@ See the reference for components utilized in this service: ### Kubernetes Control Plane Configuration -| Name | Description | Type | Value | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | -| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | -| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | -| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | -| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` | -| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | -| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | -| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | -| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | -| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | -| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | -| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | -| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` | -| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` | +| Name | Description | Type | Value | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ---------- | ----------- | +| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | +| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | +| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | +| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | +| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` | +| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | +| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | +| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | +| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | +| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | +| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | +| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | +| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` | +| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` | +| `images.kubectl` | Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag. | `string` | `""` | +| `images.talosCsrSigner` | Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag. | `string` | `""` | + + +### Talos Worker Image + +| Name | Description | Type | Value | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------ | +| `talos` | Talos worker image configuration. | `object` | `{}` | +| `talos.version` | Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. | `string` | `v1.13.0` | +| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` | + + +### Node Health Check + +| Name | Description | Type | Value | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ----- | +| `nodeHealthCheck` | MachineHealthCheck tuning for worker node groups. | `object` | `{}` | +| `nodeHealthCheck.maxUnhealthy` | Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers. | `string` | `50%` | +| `nodeHealthCheck.nodeStartupTimeout` | Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). | `string` | `10m` | ## Parameter examples and reference @@ -213,9 +235,15 @@ resources: `resourcesPreset` sets named CPU and memory configurations for each replica. This setting is ignored if the corresponding `resources` value is set. -Presets follow a cloud-style `.` naming convention. Five series cover the full CPU-to-memory ratio range (`t1` 1:0.5, `c1` 1:1, `s1` 1:2, `u1` 1:4, `m1` 1:8) and each series ships eight sizes (`nano` through `4xlarge`). The legacy flat names (`nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`) remain accepted as deprecated aliases of their 1:1 instance-type equivalents. - -See [`docs/operations/resource-presets.md`](../../../docs/operations/resource-presets.md) for the full size matrix and the legacy-to-instance-type mapping. +| Preset name | CPU | memory | +|-------------|--------|---------| +| `nano` | `250m` | `128Mi` | +| `micro` | `500m` | `256Mi` | +| `small` | `1` | `512Mi` | +| `medium` | `1` | `1Gi` | +| `large` | `2` | `2Gi` | +| `xlarge` | `4` | `4Gi` | +| `2xlarge` | `8` | `8Gi` | ### instanceType Resources diff --git a/packages/apps/kubernetes/files/versions.yaml b/packages/apps/kubernetes/files/versions.yaml index 58942b97e..86a052064 100644 --- a/packages/apps/kubernetes/files/versions.yaml +++ b/packages/apps/kubernetes/files/versions.yaml @@ -3,4 +3,3 @@ "v1.33": "v1.33.8" "v1.32": "v1.32.12" "v1.31": "v1.31.14" -"v1.30": "v1.30.14" diff --git a/packages/apps/kubernetes/images/kubectl.tag b/packages/apps/kubernetes/images/kubectl.tag new file mode 100644 index 000000000..80d1e903c --- /dev/null +++ b/packages/apps/kubernetes/images/kubectl.tag @@ -0,0 +1 @@ +docker.io/alpine/k8s:1.33.4@sha256:b0523f0a244ddc4c8e055aa335c040d3d78b3ead5528f4544395f7f9f69c7b68 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag deleted file mode 100644 index f47b874a1..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.0@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag deleted file mode 100644 index a6fae5167..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.0@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag deleted file mode 100644 index 01e417d55..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.0@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag deleted file mode 100644 index f4603e64f..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.0@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag deleted file mode 100644 index 23761242e..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.0@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag deleted file mode 100644 index 325da2bde..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag +++ /dev/null @@ -1 +0,0 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.0@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk/Dockerfile b/packages/apps/kubernetes/images/ubuntu-container-disk/Dockerfile deleted file mode 100644 index f5a7d54d6..000000000 --- a/packages/apps/kubernetes/images/ubuntu-container-disk/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -# TODO: Here we use ubuntu:22.04, as guestfish has some network issues running in ubuntu:24.04 -FROM ubuntu:22.04 AS guestfish - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update \ - && apt-get -y install \ - libguestfs-tools \ - linux-image-generic \ - wget \ - make \ - bash-completion - -WORKDIR /build - -FROM guestfish AS builder - -ARG TARGETOS -ARG TARGETARCH - -# noble is a code name for the Ubuntu 24.04 LTS release -RUN wget -O image.img https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-${TARGETARCH}.img --show-progress --output-file /dev/stdout --progress=dot:giga 2>/dev/null - -ARG KUBERNETES_VERSION - -RUN qemu-img resize image.img 5G \ - && eval "$(guestfish --listen --network)" \ - && guestfish --remote add-drive image.img \ - && guestfish --remote run \ - && guestfish --remote mount /dev/sda1 / \ - && guestfish --remote command "growpart /dev/sda 1 --verbose" \ - && guestfish --remote command "resize2fs /dev/sda1" \ -# docker repo - && guestfish --remote sh "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg" \ - && guestfish --remote sh 'echo "deb [signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list' \ -# kubernetes repo - && guestfish --remote sh "curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg" \ - && guestfish --remote sh "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list" \ - && guestfish --remote command "apt-get check -q" \ -# install containerd - && guestfish --remote command "apt-get update -q" \ - && guestfish --remote command "apt-get install -yq containerd.io" \ -# configure containerd - && guestfish --remote command "mkdir -p /etc/containerd" \ - && guestfish --remote sh "containerd config default | tee /etc/containerd/config.toml" \ - && guestfish --remote command "sed -i '/SystemdCgroup/ s/=.*/= true/' /etc/containerd/config.toml" \ - && guestfish --remote command "containerd config dump >/dev/null" \ -# install kubernetes - && guestfish --remote command "apt-get install -yq kubelet kubeadm" \ -# clean apt cache - && guestfish --remote sh 'apt-get clean && rm -rf /var/lib/apt/lists/*' \ -# write system configuration - && guestfish --remote sh 'printf "%s\n" net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.all.disable_ipv6=0 net.ipv4.tcp_congestion_control=bbr vm.overcommit_memory=1 kernel.panic=10 kernel.panic_on_oops=1 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=524288 | tee > /etc/sysctl.d/kubernetes.conf' \ - && guestfish --remote sh 'printf "%s\n" overlay br_netfilter | tee /etc/modules-load.d/kubernetes.conf' \ - && guestfish --remote sh "rm -f /etc/resolv.conf && ln -s ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf" \ -# umount all and exit - && guestfish --remote umount-all \ - && guestfish --remote exit - -FROM scratch -COPY --from=builder /build/image.img /disk/image.qcow2 diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 40a8ae7ad..659a264fa 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -50,6 +50,21 @@ app.kubernetes.io/name: {{ include "kubernetes.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{/* +DNS domain used INSIDE the tenant cluster (kubelet --cluster-domain, +apiserver --service-cluster-ip-range FQDNs, CoreDNS authoritative zone). +Pinned to Kamaji's default `networkProfile.clusterDomain` since this chart +does not currently expose a knob to override it. If that ever becomes +configurable, plumb the override here and every consumer picks it up. + +Distinct from .Values._cluster["cluster-domain"], which is the MANAGEMENT +cluster domain (e.g. cozy.local) where the Kamaji control plane and +monitoring stack live. +*/}} +{{- define "kubernetes.tenantClusterDomain" -}} +cluster.local +{{- end }} + {{/* wait-for-kubeconfig init container shared by the control-plane-side Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the @@ -59,10 +74,10 @@ Kamaji is still bootstrapping; this container polls the mounted path and exits only when super-admin.svc appears, which happens after kubelet's optional-Secret refresh cycle. -The 10m deadline stays strictly below the 15m HelmRelease +The 10m deadline stays strictly below the 20m HelmRelease Install.Timeout set by cozystack-api for the Kubernetes kind (via the -release.cozystack.io/helm-install-timeout annotation) so the -CrashLoopBackOff surfaces before flux remediation fires and uninstalls +release.cozystack.io/helm-install-timeout annotation on the cozyrds +entry) so the CrashLoopBackOff surfaces before flux remediation fires and uninstalls the Cluster CR. The default image lives in images/busybox.tag and points directly at diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 0cc8d1755..32c4a6b2a 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,3 +1,23 @@ +{{- /* + Talos -> Kubernetes support matrix. Each Talos minor release supports a + bounded window of Kubernetes minor versions; running a kubelet outside + that window produces a silently broken Talos+kubelet combination that + HelmRelease cannot detect. Fail the chart render instead, so the operator + sees the mismatch in Flux/dashboards before workers roll over. + + The matrix only fires for Talos versions it knows about; operators who + override talos.version to something outside this list take responsibility + for compatibility themselves (extend the dict or upstream a PR). + Source: https://docs.siderolabs.com/talos/v1.13/getting-started/support-matrix +*/}} +{{- $talosK8sSupportMatrix := dict + "v1.13.0" (list "v1.31" "v1.32" "v1.33" "v1.34" "v1.35" "v1.36") +}} +{{- with index $talosK8sSupportMatrix .Values.talos.version }} +{{- if not (has $.Values.version .) }} +{{- fail (printf "Kubernetes %s is not supported by Talos %s. Supported versions: %v. Bump tenant 'version' or set talos.version explicitly." $.Values.version $.Values.talos.version .) }} +{{- end }} +{{- end }} {{- $etcd := .Values._namespace.etcd }} {{- /* When $etcd is empty, the parent Tenant application has not populated @@ -10,13 +30,38 @@ flux marks the HelmRelease Ready and retries its 5m reconcile loop until the Tenant chart catches up. */}} +{{- /* The TalosConfigTemplate and the apiserver-Service ClusterIP-SAN + used to be rendered in this template behind a lookup gate that + races every fresh install: Service / Talos CA / K8s CA do not + exist on first Helm render, so the gate fails, the TCT is + skipped, and Helm's release-storage three-way merge then refuses + to upgrade because no values changed. Both objects are now + produced by the post-install/post-upgrade Helm hook Job in + templates/talos/talos-reconcile-job.yaml, which blocks until the + four dependencies exist and then `kubectl apply`s deterministic + results. The Job owns the TCT through an ownerReference to the + KamajiControlPlane so K8s garbage-collects it naturally on + release uninstall. */}} +{{- /* + Cluster pod/service CIDRs MUST match between the CAPI Cluster + spec.clusterNetwork below and the Talos worker machineconfig's + cluster.network — Talos surfaces overlaps against the host pod CIDR + as a diagnostic at boot, and divergence between CAPI and Talos would + manifest as silently broken kube-proxy / Cilium routing. Keep them + in one place here. +*/}} +{{- $podCIDR := "10.243.0.0/16" }} +{{- $serviceCIDR := "10.95.0.0/16" }} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} {{- $kubevirtmachinetemplateNames := list }} {{- define "kubevirtmachinetemplate" -}} spec: + {{- /* Talos has no SSH; use 'none' so CAPK does not block Machine.Ready + on an SSH probe that will never succeed. The Machine reaches Ready + once kubelet registers with the tenant apiserver via standard CSR. */}} virtualMachineBootstrapCheck: - checkStrategy: ssh + checkStrategy: none virtualMachineTemplate: metadata: namespace: {{ $.Release.Namespace }} @@ -33,16 +78,21 @@ spec: runStrategy: Always dataVolumeTemplates: - metadata: - name: disk-kubelet + name: disk-system annotations: - {{/* Populator creates intermediate PVC + PV rebind that causes ClaimMisbound with LINSTOR CSI */}} cdi.kubevirt.io/storage.usePopulator: "false" spec: + {{- /* Use the openstack/configdrive Talos image: KubeVirt CAPK + injects bootstrap data via cloudInitConfigDrive (OpenStack + metadata service shape), and the openstack platform image + is the one that reads it. The nocloud-amd64 image silently + ignores configdrive and never applies the machineconfig. */}} + source: + http: + url: https://factory.talos.dev/image/{{ $.Values.talos.schematicID }}/{{ $.Values.talos.version }}/openstack-amd64.raw.xz {{- if hasKey .group "ephemeralStorage" }} {{- fail (printf "nodeGroup %q: ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system." .groupName) }} {{- end }} - source: - blank: {} storage: resources: requests: @@ -92,13 +142,10 @@ spec: disk: bus: virtio pciAddress: 0000:07:00.0 - - name: disk-kubelet - disk: - bus: virtio - pciAddress: 0000:08:00.0 blockSize: - matchVolume: - enabled: true + custom: + logical: 512 + physical: 4096 interfaces: - name: default bridge: {} @@ -112,11 +159,8 @@ spec: evictionStrategy: External volumes: - name: system - containerDisk: - image: "{{ $.Files.Get (printf "images/ubuntu-container-disk-%s.tag" $.Values.version) | trim }}" - - name: disk-kubelet dataVolume: - name: disk-kubelet + name: disk-system networks: - name: default pod: {} @@ -151,10 +195,10 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.243.0.0/16 + - {{ $podCIDR }} services: cidrBlocks: - - 10.95.0.0/16 + - {{ $serviceCIDR }} controlPlaneRef: namespace: {{ .Release.Namespace }} apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 @@ -208,10 +252,69 @@ spec: nginx.ingress.kubernetes.io/ssl-passthrough: "true" hostname: {{ .Values.host | default (printf "%s.%s" .Release.Name $host) }}:443 className: "{{ $ingress }}" + additionalServicePorts: + - name: talos-trustd + port: 50001 + targetPort: 50001 + protocol: TCP + {{- /* Initial render lists only the DNS SANs because the apiserver + Service ClusterIP is not yet allocated when Helm executes the + template. The talos-reconcile post-install/post-upgrade hook + re-patches certSANs with the live ClusterIP once Kamaji emits + the Service, and Kamaji then re-issues the apiserver TLS cert + to cover it. */}} + certSANs: + - {{ printf "%s.%s.svc" .Release.Name .Release.Namespace }} + - {{ printf "%s.%s.svc.%s" .Release.Name .Release.Namespace (include "kubernetes.tenantClusterDomain" .) }} deployment: podAdditionalMetadata: labels: policy.cozystack.io/allow-to-etcd: "true" + extraContainers: + - name: talos-csr-signer + image: "{{ default (.Files.Get "images/talos-csr-signer.tag" | trim) .Values.images.talosCsrSigner }}" + imagePullPolicy: IfNotPresent + ports: + - name: trustd + containerPort: 50001 + protocol: TCP + env: + - name: TALOS_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-talos-secrets + key: token + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: talos-ca + mountPath: /etc/talos-ca + readOnly: true + - name: talos-tls-cert + mountPath: /etc/talos-server-crt + readOnly: true + extraVolumes: + - name: talos-ca + secret: + secretName: {{ .Release.Name }}-talos-ca + - name: talos-tls-cert + secret: + secretName: {{ .Release.Name }}-talos-tls-cert replicas: {{ .Values.controlPlane.replicas }} version: {{ include "kubernetes.versionMap" $ }} --- @@ -406,104 +509,18 @@ metadata: {{- end }} {{- end }} --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: {{ $.Release.Name }}-{{ $groupName }} - namespace: {{ $.Release.Namespace }} -spec: - template: - spec: - files: - - path: /usr/bin/update-k8s.sh - owner: root:root - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - # Expected to be passed in via preKubeadmCommands - : "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}" - - ARCH="$(uname -m)" - case "${ARCH}" in - x86_64) ARCH=amd64 ;; - aarch64) ARCH=arm64 ;; - esac - - # Use your internal mirror here for real-world use. - BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}" - - echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..." - curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet - curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm - chmod 0755 /root/kubelet - chmod 0755 /root/kubeadm - if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi - if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi - {{- $sec := lookup "v1" "Secret" $.Release.Namespace (printf "%s-patch-containerd" $.Release.Name) }} - {{- if $sec }} - {{- range $key, $_ := $sec.data }} - - path: /etc/containerd/certs.d/{{ trimSuffix ".toml" $key }}/hosts.toml - contentFrom: - secret: - name: {{ $.Release.Name }}-patch-containerd - key: {{ $key }} - permissions: "0400" - {{- end }} - {{- end }} - diskSetup: - filesystems: - - device: /dev/vdb - filesystem: xfs - label: persistent - partition: "none" - {{/* overwrite must stay false: XFS on /dev/vdb is created on first boot; subsequent reboots of the same VM must reuse the existing filesystem via LABEL=persistent */}} - overwrite: false - mounts: - - ["LABEL=persistent", "/persistent"] - - ["/persistent/kubelet", "/var/lib/kubelet", "none", "bind,nofail"] - - ["/persistent/containerd", "/var/lib/containerd", "none", "bind,nofail"] - preKubeadmCommands: - - KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true - - sed -i 's|root:x:|root::|' /etc/passwd - - systemctl stop containerd.service - - mkdir -p /persistent/kubelet /persistent/containerd /var/lib/kubelet /var/lib/containerd - - mountpoint -q /var/lib/kubelet || mount -o bind /persistent/kubelet /var/lib/kubelet - - mountpoint -q /var/lib/containerd || mount -o bind /persistent/containerd /var/lib/containerd - {{/* containerd 2.x renamed the CRI plugin from io.containerd.grpc.v1.cri to io.containerd.cri.v1.images; both seds run unconditionally and the wrong-version one is a no-op */}} - - sed -i '/\[plugins\.[^]]*grpc\.v1\.cri[^]]*registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml - - sed -i '/\[plugins\.[^]]*io\.containerd\.cri\.v1\.images[^]]*registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml - - systemctl start containerd.service - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - {{- /* - GPU node-groups: label every node `gpu=on` so HAMi's - hami-device-plugin DaemonSet (nodeSelector gpu=on) schedules - and advertises nvidia.com/gpu. Without the label the plugin - stays at DESIRED=0 and no GPUs are exposed to the tenant. - */}} - {{- if $group.gpus }} - node-labels: "gpu=on" - {{- end }} - system-reserved: "memory={{ $systemReservedMemory }},cpu={{ $systemReservedCpu }}" - kube-reserved: "memory={{ $kubeReservedMemory }},cpu={{ $kubeReservedCpu }}" - eviction-hard: "memory.available<{{ $evictionHardMemory }},nodefs.available<10%,imagefs.available<15%,nodefs.inodesFree<5%" - eviction-soft: "memory.available<{{ $evictionSoftMemory }},nodefs.available<15%,imagefs.available<20%" - eviction-soft-grace-period: "memory.available=1m30s,nodefs.available=1m30s,imagefs.available=1m30s" - # 256Mi ensures meaningful reclamation per eviction cycle while being - # proportional to the 256Mi-1Gi auto-computed reservation range. - eviction-minimum-reclaim: "memory.available=256Mi" - enforce-node-allocatable: "pods" - ignorePreflightErrors: - - FileExisting-conntrack - discovery: - bootstrapToken: - apiServerEndpoint: {{ $.Release.Name }}.{{ $.Release.Namespace }}.svc:6443 - initConfiguration: - skipPhases: - - addon/kube-proxy +{{- /* TalosConfigTemplate is no longer rendered here — the chart cannot + produce a valid Talos worker machineconfig at Helm template time + because three of the inputs (apiserver Service ClusterIP, Talos + CA from cert-manager, Kamaji-issued tenant CA) are created + asynchronously after Helm applies this manifest. The + talos-reconcile-job hook (templates/talos/talos-reconcile-job.yaml) + waits for those inputs and produces the TalosConfigTemplate with + a KamajiControlPlane ownerReference for natural GC. The + MachineDeployment below still references the TCT name; CAPI's + Machine controller blocks on its absence and unblocks the moment + the hook applies it, which happens in the same post-install/ + post-upgrade cycle. */}} --- {{- $context := deepCopy $ }} {{- $_ := set $context "group" $group }} @@ -569,8 +586,8 @@ spec: spec: bootstrap: configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: TalosConfigTemplate name: {{ $.Release.Name }}-{{ $groupName }} namespace: {{ $.Release.Namespace }} clusterName: {{ $.Release.Name }} @@ -588,8 +605,13 @@ metadata: namespace: {{ $.Release.Namespace }} spec: clusterName: {{ $.Release.Name }} - maxUnhealthy: 0 - nodeStartupTimeout: 10m + {{- /* MHC spec.maxUnhealthy is an IntOrString. CAPI admission accepts a + bare integer (0, 1, ...) or a percentage string ("0%", "50%"), but + rejects a quoted plain integer ("0"). Quote only when the operator + supplied a percentage; otherwise render the value as a bare int so + the admission webhook is happy with the chart-default "0". */}} + maxUnhealthy: {{ if hasSuffix "%" (toString $.Values.nodeHealthCheck.maxUnhealthy) }}{{ $.Values.nodeHealthCheck.maxUnhealthy | quote }}{{ else }}{{ $.Values.nodeHealthCheck.maxUnhealthy | int }}{{ end }} + nodeStartupTimeout: {{ $.Values.nodeHealthCheck.nodeStartupTimeout }} selector: matchLabels: cluster.x-k8s.io/cluster-name: {{ $.Release.Name }} diff --git a/packages/apps/kubernetes/templates/copy-patch-containerd.yaml b/packages/apps/kubernetes/templates/copy-patch-containerd.yaml deleted file mode 100644 index 433452763..000000000 --- a/packages/apps/kubernetes/templates/copy-patch-containerd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- $sourceSecret := lookup "v1" "Secret" "cozy-system" "patch-containerd" }} -{{- if $sourceSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Release.Name }}-patch-containerd - namespace: {{ .Release.Namespace }} -type: {{ $sourceSecret.type }} -data: -{{- range $key, $value := $sourceSecret.data }} - {{ printf "%s: %s" $key ($value | quote) | indent 2 }} -{{- end }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml index 5b69e2f29..9ee046799 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml @@ -1,6 +1,14 @@ {{- define "cozystack.defaultCiliumValues" -}} +{{- /* Cilium runs hostNetwork on the Talos worker nodes and resolves the + apiserver via the worker's /etc/hosts, which the talos-reconcile + post-install hook seeds with the Kamaji Service ClusterIP under + ..svc through machine.network.extraHostEntries. So a + plain DNS host works here without a lookup-and-reuse race: the + cert covers the same DNS name from chart render time, and the + hosts entry is set well before kubelet (and therefore Cilium) + comes up on the node. */}} cilium: - k8sServiceHost: {{ .Release.Name }}.{{ .Release.Namespace }}.svc + k8sServiceHost: {{ printf "%s.%s.svc" .Release.Name .Release.Namespace }} k8sServicePort: 6443 routingMode: tunnel MTU: 1350 diff --git a/packages/apps/kubernetes/templates/helmreleases/ouroboros.yaml b/packages/apps/kubernetes/templates/helmreleases/ouroboros.yaml index 09c20aa05..ecc8c8aae 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ouroboros.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ouroboros.yaml @@ -41,7 +41,7 @@ ouroboros: federations, custom kubelet --cluster-domain, etc.) can override via `addons.ouroboros.valuesOverride.ouroboros.controller.clusterDomain`. */}} - clusterDomain: cluster.local + clusterDomain: {{ include "kubernetes.tenantClusterDomain" $ }} {{- end }} {{- /* addons.ouroboros.enabled requires addons.ingressNginx.enabled. The diff --git a/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml b/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml new file mode 100644 index 000000000..49902ea91 --- /dev/null +++ b/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml @@ -0,0 +1,152 @@ +{{- /* + Job that materialises a long-lived kubeadm-format bootstrap-token Secret + inside the tenant kube-system namespace, so that Talos workers can use + cluster.token from their machineconfig to perform kubelet CSR. + + Why a Job and not a kube-native Secret on the management cluster: + bootstrap-token- Secrets must live in the tenant apiserver's + kube-system, which the management cluster can only reach via the + Kamaji-issued admin kubeconfig stored as -admin-kubeconfig. + + The Job runs as a Helm post-install/post-upgrade hook so it executes + after the KamajiControlPlane has reconciled and the admin kubeconfig + Secret exists. It is idempotent (kubectl apply) and re-runs harmlessly + on every upgrade. +*/}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-bootstrap-token-tenant + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-bootstrap-token-tenant + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + # backoffLimit + restartPolicy: OnFailure gives the Job up to 11 attempts at + # the tenant apiserver with exponential backoff (capped at 6 min between + # restarts). Worst case ~15 min, well inside the chart's HR install timeout + # (release.cozystack.io/helm-install-timeout: 20m on the cozyrds entry) so + # the Job either succeeds within the HR window or fails the chart upgrade + # loudly, instead of letting Helm mark the install "successful" while the + # bootstrap-token Secret never lands in the tenant kube-system. + backoffLimit: 10 + ttlSecondsAfterFinished: 600 + template: + spec: + restartPolicy: OnFailure + serviceAccountName: {{ .Release.Name }}-bootstrap-token-tenant + containers: + - name: kubectl + image: "{{ default (.Files.Get "images/kubectl.tag" | trim) .Values.images.kubectl }}" + env: + - name: KUBECONFIG + value: /tenant-admin/super-admin.svc + - name: TOKEN_ID + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-talos-secrets + key: bootstrapTokenId + - name: TOKEN_SECRET + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-talos-secrets + key: bootstrapTokenSecret + volumeMounts: + - name: tenant-admin + mountPath: /tenant-admin + readOnly: true + command: + - sh + - -ec + - | + # The kubectl apply calls below fail with a non-zero exit when + # the tenant apiserver is unreachable, which restarts the pod + # under backoffLimit. No internal retry loop needed — Job-level + # backoff handles the transient unreadiness window after Kamaji + # bootstraps the konnectivity plane. + kubectl version --request-timeout=10s + + cat <..svc") + the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The + kubevirt-csi NFS mount path is the load-bearing case: kubelet on + the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc` + (no domain suffix), and without searchDomains in /etc/resolv.conf + the host resolver hands that name to management CoreDNS verbatim + and gets NXDOMAIN. + + Distinct from tenantClusterDomain ("cluster.local"), which is the + *tenant* kubelet --cluster-dns zone. This value is the + *management* cluster-domain, sourced from .Values._cluster + (populated by cozystack-controller from the platform config). */}} +{{- $mgmtClusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} +{{- range $groupName, $group := .Values.nodeGroups }} +{{- /* Mirror the kubelet-reservation computation from cluster.yaml so the + worker machineconfig the hook applies carries the same numbers the + MachineDeployment validator was happy with. Auto-computed + system/kube reserved memory = 5% of effective memory clamped to + [256Mi, 1Gi]; auto-computed system/kube reserved CPU = 5% of + effective CPU clamped to [50m, 500m]. instanceType lookup is + optional — operators that override resources.cpu / resources.memory + on the nodeGroup get those, otherwise the chart falls back to the + same static-defaults pair the cluster.yaml validator uses on a + no-instanceType setup. */}} +{{- $instanceType := dict }} +{{- if $group.instanceType }} +{{- $instanceType = lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType }} +{{- $instanceType = $instanceType | default dict }} +{{- end }} +{{- $effectiveMemory := "" }} +{{- if and $group.resources $group.resources.memory }} +{{- $effectiveMemory = $group.resources.memory | toString }} +{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} +{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} +{{- end }} +{{- $effectiveCpu := "" }} +{{- if and $group.resources $group.resources.cpu }} +{{- $effectiveCpu = $group.resources.cpu | toString }} +{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} +{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} +{{- end }} +{{- $autoReservedMi := 256 }} +{{- if $effectiveMemory }} +{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} +{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} +{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} +{{- end }} +{{- $autoReservedMillicores := 50 }} +{{- if $effectiveCpu }} +{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} +{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }} +{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} +{{- end }} +{{- $kubeletOverride := $group.kubelet | default dict }} +{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} +{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} +{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} +{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} +{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} +{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $.Release.Name }}-talos-reconcile-{{ $groupName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "kubernetes.labels" $ | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 5 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + cozystack.io/talos-reconcile: {{ $.Release.Name | quote }} + spec: + restartPolicy: OnFailure + serviceAccountName: {{ $.Release.Name }}-talos-reconcile + containers: + - name: kubectl + image: "{{ default ($.Files.Get "images/kubectl.tag" | trim) $.Values.images.kubectl }}" + env: + - name: RELEASE + value: {{ $.Release.Name | quote }} + - name: NS + value: {{ $.Release.Namespace | quote }} + - name: GROUP_NAME + value: {{ $groupName | quote }} + - name: TALOS_TOKEN + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-talos-secrets + key: token + - name: CLUSTER_ID + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-talos-secrets + key: clusterId + - name: CLUSTER_SECRET + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-talos-secrets + key: clusterSecret + - name: BOOTSTRAP_TOKEN + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-talos-secrets + key: bootstrapToken + command: + - sh + - -ec + - | + echo "[$(date -u +%H:%M:%S)] talos-reconcile/${GROUP_NAME}: waiting for Service ClusterIP, Talos CA, K8s CA, management CoreDNS" + for i in $(seq 1 120); do + SVC_IP=$(kubectl -n "$NS" get svc "$RELEASE" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true) + TALOS_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-talos-ca" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true) + K8S_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true) + COREDNS_IP=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true) + if [ -n "$SVC_IP" ] && [ -n "$TALOS_CA_B64" ] && [ -n "$K8S_CA_B64" ] && [ -n "$COREDNS_IP" ]; then + break + fi + sleep 5 + done + if [ -z "$SVC_IP" ] || [ -z "$TALOS_CA_B64" ] || [ -z "$K8S_CA_B64" ] || [ -z "$COREDNS_IP" ]; then + echo "[$(date -u +%H:%M:%S)] timed out waiting for lookup inputs (svc='$SVC_IP' talos-ca=$([ -n "$TALOS_CA_B64" ] && echo ok) k8s-ca=$([ -n "$K8S_CA_B64" ] && echo ok) coredns='$COREDNS_IP')" + exit 1 + fi + echo "[$(date -u +%H:%M:%S)] inputs ready (svc=${SVC_IP} coredns=${COREDNS_IP})" + + # Patch KamajiControlPlane.spec.network.certSANs so the apiserver + # TLS cert covers the live Service ClusterIP that Talos workers + # dial. JSON-merge-patch is array-replace semantics, which is what + # we want here — Kamaji owns the full SAN list. + kubectl -n "$NS" patch kamajicontrolplane "$RELEASE" --type=merge --patch "$( + cat <Kubernetes support matrix against the chosen `version`. +## @field {string} schematicID - Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. + +## @param {Talos} talos - Talos worker image configuration. +talos: + version: "v1.13.0" + schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" + +## +## @section Node Health Check +## + +## @typedef {struct} NodeHealthCheck - MachineHealthCheck configuration for worker node groups. +## @field {string} maxUnhealthy="50%" - Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers. +## @field {string} nodeStartupTimeout="10m" - Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). + +## @param {NodeHealthCheck} nodeHealthCheck - MachineHealthCheck tuning for worker node groups. +nodeHealthCheck: + maxUnhealthy: "50%" + nodeStartupTimeout: "10m" diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 9ce179839..a275dd667 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -15,14 +15,24 @@ metadata: # deadline chosen to stay strictly below this value so the pod's # CrashLoopBackOff surfaces before flux remediation fires. If this # annotation is raised, update that init deadline correspondingly. - release.cozystack.io/helm-install-timeout: "15m" + release.cozystack.io/helm-install-timeout: "20m" + # The kubernetes chart emits in-tenant addon HelmReleases (cilium, coredns, + # csi, ingress-nginx, metrics-server, ouroboros) and ships a talos-reconcile + # post-install hook that produces the TalosConfigTemplate the worker + # MachineSets clone from. Without DisableWait the helm-controller waits for + # the addon HelmReleases to be Ready before running the hook, but those + # cannot be Ready until worker nodes exist, and the workers cannot be + # created without the TalosConfigTemplate the hook is supposed to produce. + # Skip the readiness wait so the hook fires and the chicken-and-egg + # resolves. + release.cozystack.io/helm-install-disable-wait: "true" spec: application: kind: Kubernetes singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"kubectl":{"description":"Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.","type":"string","default":""},"talosCsrSigner":{"description":"Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.","type":"string","default":""},"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}},"talos":{"description":"Talos worker image configuration.","type":"object","default":{},"required":["schematicID","version"],"properties":{"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.","type":"string","default":"v1.13.0"}}},"nodeHealthCheck":{"description":"MachineHealthCheck tuning for worker node groups.","type":"object","default":{},"required":["maxUnhealthy","nodeStartupTimeout"],"properties":{"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default \"50%\" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to \"0%\" once the fleet is stable on Talos workers.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).","type":"string","default":"10m"}}}}} release: prefix: kubernetes- labels: @@ -40,7 +50,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "diskSize"], ["spec", "nodeGroups", "md0", "storageClass"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "nodeGroups", "md0", "kubelet"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "diskSize"], ["spec", "nodeGroups", "md0", "storageClass"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "nodeGroups", "md0", "kubelet"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"], ["spec", "images", "kubectl"], ["spec", "images", "talosCsrSigner"], ["spec", "talos"], ["spec", "talos", "version"], ["spec", "talos", "schematicID"], ["spec", "nodeHealthCheck"], ["spec", "nodeHealthCheck", "maxUnhealthy"], ["spec", "nodeHealthCheck", "nodeStartupTimeout"]] secrets: exclude: [] include: