feat(kubernetes): bootstrap tenant workers with Talos via CABPT

Switch tenant worker bootstrap from Ubuntu+kubeadm to Talos. MachineDeployment.bootstrap moves from KubeadmConfigTemplate to a TalosConfigTemplate produced by a post-install/upgrade reconcile hook, which also materialises the kubeadm-format bootstrap-token Secret in the tenant kube-system. Workers boot from the Talos openstack image via a CDI DataVolume; the system disk is virtio-blk with logical=512/physical=4096 so 4Ki-native backends (LINSTOR/DRBD) and SeaBIOS both work; the separate kubelet disk is dropped. Pod/service CIDRs, dnsDomain, controlPlane endpoint and cilium k8sServiceHost are pinned to the tenant ranges / Kamaji apiserver ClusterIP to survive the DNS chicken-and-egg at bootstrap. Adds a Talos<->Kubernetes support-matrix fail() guard and drops v1.30. Removes the dead ubuntu-container-disk build chain and the orphaned copy-patch-containerd template.

Co-authored-by: IvanHunters <xorokhotnikov@gmail.com>
Co-authored-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
authored and Myasnikov Daniil committed 2026-06-23 14:31:16 +05:00
1 parent ccf6d9dbfe
commit 39cdc88661
23 files changed
+1049 -261

No files matched your search

+32 -2
View File
@@ -40,6 +40,12 @@ type ConfigSpec struct {
// Optional image overrides for air-gapped or rate-limited registries.
// +kubebuilder:default:={}
Images Images `json:"images"`
// Talos worker image configuration.
// +kubebuilder:default:={}
Talos Talos `json:"talos"`
// MachineHealthCheck tuning for worker node groups.
// +kubebuilder:default:={}
NodeHealthCheck NodeHealthCheck `json:"nodeHealthCheck"`
}
type APIServer struct {
@@ -177,6 +183,12 @@ type HAMiAddon struct {
}
type Images struct {
// Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.
// +kubebuilder:default:=""
Kubectl string `json:"kubectl,omitempty"`
// Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.
// +kubebuilder:default:=""
TalosCsrSigner string `json:"talosCsrSigner,omitempty"`
// Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
// +kubebuilder:default:=""
WaitForKubeconfig string `json:"waitForKubeconfig,omitempty"`
@@ -239,7 +251,7 @@ type MonitoringAgentsAddon struct {
}
type NodeGroup struct {
// Persistent disk size for kubelet and containerd data.
// System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.
// +kubebuilder:default:="20Gi"
DiskSize resource.Quantity `json:"diskSize"`
// List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
@@ -263,6 +275,15 @@ type NodeGroup struct {
StorageClass string `json:"storageClass,omitempty"`
}
type NodeHealthCheck struct {
// Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers.
// +kubebuilder:default:="50%"
MaxUnhealthy string `json:"maxUnhealthy"`
// Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).
// +kubebuilder:default:="10m"
NodeStartupTimeout string `json:"nodeStartupTimeout"`
}
type OuroborosAddon struct {
// Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.
// +kubebuilder:default:=false
@@ -288,6 +309,15 @@ type Scheduler struct {
ResourcesPreset ResourcesPreset `json:"resourcesPreset"`
}
type Talos struct {
// Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.
// +kubebuilder:default:="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
SchematicID string `json:"schematicID"`
// Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.
// +kubebuilder:default:="v1.13.0"
Version string `json:"version"`
}
type VeleroAddon struct {
// Enable Velero.
// +kubebuilder:default:=false
@@ -309,5 +339,5 @@ type IngressNginxExposeMethod string
// +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge"
type ResourcesPreset string
// +kubebuilder:validation:Enum="v1.35";"v1.34";"v1.33";"v1.32";"v1.31";"v1.30"
// +kubebuilder:validation:Enum="v1.35";"v1.34";"v1.33";"v1.32";"v1.31"
type Version string
@@ -138,6 +138,8 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
in.Addons.DeepCopyInto(&out.Addons)
in.ControlPlane.DeepCopyInto(&out.ControlPlane)
out.Images = in.Images
out.Talos = in.Talos
out.NodeHealthCheck = in.NodeHealthCheck
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec.
@@ -406,6 +408,21 @@ func (in *NodeGroup) DeepCopy() *NodeGroup {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeHealthCheck) DeepCopyInto(out *NodeHealthCheck) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeHealthCheck.
func (in *NodeHealthCheck) DeepCopy() *NodeHealthCheck {
if in == nil {
return nil
}
out := new(NodeHealthCheck)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OuroborosAddon) DeepCopyInto(out *OuroborosAddon) {
*out = *in
@@ -455,6 +472,21 @@ func (in *Scheduler) DeepCopy() *Scheduler {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Talos) DeepCopyInto(out *Talos) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Talos.
func (in *Talos) DeepCopy() *Talos {
if in == nil {
return nil
}
out := new(Talos)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *VeleroAddon) DeepCopyInto(out *VeleroAddon) {
*out = *in
+10 -15
View File
@@ -8,7 +8,6 @@ include ../../../hack/package.mk
test:
helm unittest .
bash tests/containerd-config-path-test.sh
generate:
cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go
@@ -18,21 +17,17 @@ update:
hack/update-versions.sh
make generate
image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler
image: image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-talos-csr-signer
image-ubuntu-container-disk:
$(foreach ver,$(KUBERNETES_VERSIONS), \
docker buildx build images/ubuntu-container-disk \
--build-arg KUBERNETES_VERSION=$(ver) \
--tag $(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG) \
$(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$(ver)) \
$(call cache-args,ubuntu-container-disk,$(ver)-buildcache) \
--metadata-file images/ubuntu-container-disk-$(ver).json \
$(BUILDX_ARGS) && \
echo "$(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$(ver).json -o json -r)" \
> images/ubuntu-container-disk-$(ver).tag && \
rm -f images/ubuntu-container-disk-$(ver).json; \
)
image-talos-csr-signer:
docker buildx build images/talos-csr-signer \
$(call image-tags,talos-csr-signer,$(KUBERNETES_PKG_TAG)) \
$(call cache-args,talos-csr-signer) \
--metadata-file images/talos-csr-signer.json \
$(BUILDX_ARGS)
echo "$(REGISTRY)/talos-csr-signer:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/talos-csr-signer.json -o json -r)" \
> images/talos-csr-signer.tag
rm -f images/talos-csr-signer.json
image-kubevirt-cloud-provider:
docker buildx build images/kubevirt-cloud-provider \
+60 -32
View File
@@ -85,7 +85,9 @@ See the reference for components utilized in this service:
## Breaking Changes
- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize` to better reflect its purpose (persistent disk for kubelet and containerd data). Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs will be automatically rolling-updated via CAPI when the new values are applied. State persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh PVC.
- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize`. Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs are rolling-updated via CAPI when the new values are applied. With the Talos worker rollover in this release the field now sizes the **single system disk** (Talos OS image streamed from `factory.talos.dev`, kubelet state, containerd image cache, local-path PVCs) — the pre-Talos `disk-kubelet` PVC layout was removed. State on that single disk persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh disk.
- **Air-gapped tenant workers temporarily unsupported (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos image fetched over HTTPS from `factory.talos.dev` and a `factory.talos.dev/installer/...` installer reference, neither of which is currently overridable. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig and was removed in this release. Tenants behind a strict-egress firewall or relying on `registries.mirrors` for tenant workers will be unable to create new workers until Phase 2 restores this via `machine.registries.mirrors` knobs and an installable/imageBase override. Tracked as a follow-up to this Phase 1 PR — file an issue if you depend on this and it is not yet landed.
> The top-level `storageClass` field is annotated as immutable in the chart schema — see [`docs/storage-immutability.md`](../../../docs/storage-immutability.md) for the contract and which consumers enforce it. The per-node-group `nodeGroups[name].storageClass` field is intentionally **not** annotated immutable: it is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.
@@ -106,7 +108,7 @@ See the reference for components utilized in this service:
| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` |
| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` |
| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` |
| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` |
| `nodeGroups[name].diskSize` | System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact. | `quantity` | `20Gi` |
| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` |
| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` |
| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` |
@@ -168,33 +170,53 @@ See the reference for components utilized in this service:
### Kubernetes Control Plane Configuration
| Name | Description | Type | Value |
| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ----------- |
| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` |
| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` |
| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` |
| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` |
| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` |
| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` |
| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` |
| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` |
| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` |
| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` |
| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` |
| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` |
| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` |
| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` |
| Name | Description | Type | Value |
| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ---------- | ----------- |
| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` |
| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` |
| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` |
| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` |
| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` |
| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` |
| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` |
| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` |
| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` |
| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` |
| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` |
| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` |
| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` |
| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` |
| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` |
| `images.kubectl` | Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag. | `string` | `""` |
| `images.talosCsrSigner` | Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag. | `string` | `""` |
### Talos Worker Image
| Name | Description | Type | Value |
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------ |
| `talos` | Talos worker image configuration. | `object` | `{}` |
| `talos.version` | Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. | `string` | `v1.13.0` |
| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` |
### Node Health Check
| Name | Description | Type | Value |
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ----- |
| `nodeHealthCheck` | MachineHealthCheck tuning for worker node groups. | `object` | `{}` |
| `nodeHealthCheck.maxUnhealthy` | Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers. | `string` | `50%` |
| `nodeHealthCheck.nodeStartupTimeout` | Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). | `string` | `10m` |
## Parameter examples and reference
@@ -213,9 +235,15 @@ resources:
`resourcesPreset` sets named CPU and memory configurations for each replica.
This setting is ignored if the corresponding `resources` value is set.
Presets follow a cloud-style `<series>.<size>` naming convention. Five series cover the full CPU-to-memory ratio range (`t1` 1:0.5, `c1` 1:1, `s1` 1:2, `u1` 1:4, `m1` 1:8) and each series ships eight sizes (`nano` through `4xlarge`). The legacy flat names (`nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`) remain accepted as deprecated aliases of their 1:1 instance-type equivalents.
See [`docs/operations/resource-presets.md`](../../../docs/operations/resource-presets.md) for the full size matrix and the legacy-to-instance-type mapping.
| Preset name | CPU | memory |
|-------------|--------|---------|
| `nano` | `250m` | `128Mi` |
| `micro` | `500m` | `256Mi` |
| `small` | `1` | `512Mi` |
| `medium` | `1` | `1Gi` |
| `large` | `2` | `2Gi` |
| `xlarge` | `4` | `4Gi` |
| `2xlarge` | `8` | `8Gi` |
### instanceType Resources
@@ -3,4 +3,3 @@
"v1.33": "v1.33.8"
"v1.32": "v1.32.12"
"v1.31": "v1.31.14"
"v1.30": "v1.30.14"
@@ -0,0 +1 @@
docker.io/alpine/k8s:1.33.4@sha256:b0523f0a244ddc4c8e055aa335c040d3d78b3ead5528f4544395f7f9f69c7b68
@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.0@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd
@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.0@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70
@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.0@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9
@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.0@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184
@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.0@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574
@@ -1 +0,0 @@
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.0@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2
@@ -1,60 +0,0 @@
# TODO: Here we use ubuntu:22.04, as guestfish has some network issues running in ubuntu:24.04
FROM ubuntu:22.04 AS guestfish
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get -y install \
libguestfs-tools \
linux-image-generic \
wget \
make \
bash-completion
WORKDIR /build
FROM guestfish AS builder
ARG TARGETOS
ARG TARGETARCH
# noble is a code name for the Ubuntu 24.04 LTS release
RUN wget -O image.img https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-${TARGETARCH}.img --show-progress --output-file /dev/stdout --progress=dot:giga 2>/dev/null
ARG KUBERNETES_VERSION
RUN qemu-img resize image.img 5G \
&& eval "$(guestfish --listen --network)" \
&& guestfish --remote add-drive image.img \
&& guestfish --remote run \
&& guestfish --remote mount /dev/sda1 / \
&& guestfish --remote command "growpart /dev/sda 1 --verbose" \
&& guestfish --remote command "resize2fs /dev/sda1" \
# docker repo
&& guestfish --remote sh "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg" \
&& guestfish --remote sh 'echo "deb [signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list' \
# kubernetes repo
&& guestfish --remote sh "curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg" \
&& guestfish --remote sh "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list" \
&& guestfish --remote command "apt-get check -q" \
# install containerd
&& guestfish --remote command "apt-get update -q" \
&& guestfish --remote command "apt-get install -yq containerd.io" \
# configure containerd
&& guestfish --remote command "mkdir -p /etc/containerd" \
&& guestfish --remote sh "containerd config default | tee /etc/containerd/config.toml" \
&& guestfish --remote command "sed -i '/SystemdCgroup/ s/=.*/= true/' /etc/containerd/config.toml" \
&& guestfish --remote command "containerd config dump >/dev/null" \
# install kubernetes
&& guestfish --remote command "apt-get install -yq kubelet kubeadm" \
# clean apt cache
&& guestfish --remote sh 'apt-get clean && rm -rf /var/lib/apt/lists/*' \
# write system configuration
&& guestfish --remote sh 'printf "%s\n" net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.all.disable_ipv6=0 net.ipv4.tcp_congestion_control=bbr vm.overcommit_memory=1 kernel.panic=10 kernel.panic_on_oops=1 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=524288 | tee > /etc/sysctl.d/kubernetes.conf' \
&& guestfish --remote sh 'printf "%s\n" overlay br_netfilter | tee /etc/modules-load.d/kubernetes.conf' \
&& guestfish --remote sh "rm -f /etc/resolv.conf && ln -s ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf" \
# umount all and exit
&& guestfish --remote umount-all \
&& guestfish --remote exit
FROM scratch
COPY --from=builder /build/image.img /disk/image.qcow2
@@ -50,6 +50,21 @@ app.kubernetes.io/name: {{ include "kubernetes.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
DNS domain used INSIDE the tenant cluster (kubelet --cluster-domain,
apiserver --service-cluster-ip-range FQDNs, CoreDNS authoritative zone).
Pinned to Kamaji's default `networkProfile.clusterDomain` since this chart
does not currently expose a knob to override it. If that ever becomes
configurable, plumb the override here and every consumer picks it up.
Distinct from .Values._cluster["cluster-domain"], which is the MANAGEMENT
cluster domain (e.g. cozy.local) where the Kamaji control plane and
monitoring stack live.
*/}}
{{- define "kubernetes.tenantClusterDomain" -}}
cluster.local
{{- end }}
{{/*
wait-for-kubeconfig init container shared by the control-plane-side
Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the
@@ -59,10 +74,10 @@ Kamaji is still bootstrapping; this container polls the mounted path and
exits only when super-admin.svc appears, which happens after kubelet's
optional-Secret refresh cycle.
The 10m deadline stays strictly below the 15m HelmRelease
The 10m deadline stays strictly below the 20m HelmRelease
Install.Timeout set by cozystack-api for the Kubernetes kind (via the
release.cozystack.io/helm-install-timeout annotation) so the
CrashLoopBackOff surfaces before flux remediation fires and uninstalls
release.cozystack.io/helm-install-timeout annotation on the cozyrds
entry) so the CrashLoopBackOff surfaces before flux remediation fires and uninstalls
the Cluster CR.
The default image lives in images/busybox.tag and points directly at
+141 -119
View File
@@ -1,3 +1,23 @@
{{- /*
Talos -> Kubernetes support matrix. Each Talos minor release supports a
bounded window of Kubernetes minor versions; running a kubelet outside
that window produces a silently broken Talos+kubelet combination that
HelmRelease cannot detect. Fail the chart render instead, so the operator
sees the mismatch in Flux/dashboards before workers roll over.
The matrix only fires for Talos versions it knows about; operators who
override talos.version to something outside this list take responsibility
for compatibility themselves (extend the dict or upstream a PR).
Source: https://docs.siderolabs.com/talos/v1.13/getting-started/support-matrix
*/}}
{{- $talosK8sSupportMatrix := dict
"v1.13.0" (list "v1.31" "v1.32" "v1.33" "v1.34" "v1.35" "v1.36")
}}
{{- with index $talosK8sSupportMatrix .Values.talos.version }}
{{- if not (has $.Values.version .) }}
{{- fail (printf "Kubernetes %s is not supported by Talos %s. Supported versions: %v. Bump tenant 'version' or set talos.version explicitly." $.Values.version $.Values.talos.version .) }}
{{- end }}
{{- end }}
{{- $etcd := .Values._namespace.etcd }}
{{- /*
When $etcd is empty, the parent Tenant application has not populated
@@ -10,13 +30,38 @@
flux marks the HelmRelease Ready and retries its 5m reconcile loop until
the Tenant chart catches up.
*/}}
{{- /* The TalosConfigTemplate and the apiserver-Service ClusterIP-SAN
used to be rendered in this template behind a lookup gate that
races every fresh install: Service / Talos CA / K8s CA do not
exist on first Helm render, so the gate fails, the TCT is
skipped, and Helm's release-storage three-way merge then refuses
to upgrade because no values changed. Both objects are now
produced by the post-install/post-upgrade Helm hook Job in
templates/talos/talos-reconcile-job.yaml, which blocks until the
four dependencies exist and then `kubectl apply`s deterministic
results. The Job owns the TCT through an ownerReference to the
KamajiControlPlane so K8s garbage-collects it naturally on
release uninstall. */}}
{{- /*
Cluster pod/service CIDRs MUST match between the CAPI Cluster
spec.clusterNetwork below and the Talos worker machineconfig's
cluster.network — Talos surfaces overlaps against the host pod CIDR
as a diagnostic at boot, and divergence between CAPI and Talos would
manifest as silently broken kube-proxy / Cilium routing. Keep them
in one place here.
*/}}
{{- $podCIDR := "10.243.0.0/16" }}
{{- $serviceCIDR := "10.95.0.0/16" }}
{{- $ingress := .Values._namespace.ingress }}
{{- $host := .Values._namespace.host }}
{{- $kubevirtmachinetemplateNames := list }}
{{- define "kubevirtmachinetemplate" -}}
spec:
{{- /* Talos has no SSH; use 'none' so CAPK does not block Machine.Ready
on an SSH probe that will never succeed. The Machine reaches Ready
once kubelet registers with the tenant apiserver via standard CSR. */}}
virtualMachineBootstrapCheck:
checkStrategy: ssh
checkStrategy: none
virtualMachineTemplate:
metadata:
namespace: {{ $.Release.Namespace }}
@@ -33,16 +78,21 @@ spec:
runStrategy: Always
dataVolumeTemplates:
- metadata:
name: disk-kubelet
name: disk-system
annotations:
{{/* Populator creates intermediate PVC + PV rebind that causes ClaimMisbound with LINSTOR CSI */}}
cdi.kubevirt.io/storage.usePopulator: "false"
spec:
{{- /* Use the openstack/configdrive Talos image: KubeVirt CAPK
injects bootstrap data via cloudInitConfigDrive (OpenStack
metadata service shape), and the openstack platform image
is the one that reads it. The nocloud-amd64 image silently
ignores configdrive and never applies the machineconfig. */}}
source:
http:
url: https://factory.talos.dev/image/{{ $.Values.talos.schematicID }}/{{ $.Values.talos.version }}/openstack-amd64.raw.xz
{{- if hasKey .group "ephemeralStorage" }}
{{- fail (printf "nodeGroup %q: ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system." .groupName) }}
{{- end }}
source:
blank: {}
storage:
resources:
requests:
@@ -92,13 +142,10 @@ spec:
disk:
bus: virtio
pciAddress: 0000:07:00.0
- name: disk-kubelet
disk:
bus: virtio
pciAddress: 0000:08:00.0
blockSize:
matchVolume:
enabled: true
custom:
logical: 512
physical: 4096
interfaces:
- name: default
bridge: {}
@@ -112,11 +159,8 @@ spec:
evictionStrategy: External
volumes:
- name: system
containerDisk:
image: "{{ $.Files.Get (printf "images/ubuntu-container-disk-%s.tag" $.Values.version) | trim }}"
- name: disk-kubelet
dataVolume:
name: disk-kubelet
name: disk-system
networks:
- name: default
pod: {}
@@ -151,10 +195,10 @@ spec:
clusterNetwork:
pods:
cidrBlocks:
- 10.243.0.0/16
- {{ $podCIDR }}
services:
cidrBlocks:
- 10.95.0.0/16
- {{ $serviceCIDR }}
controlPlaneRef:
namespace: {{ .Release.Namespace }}
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
@@ -208,10 +252,69 @@ spec:
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
hostname: {{ .Values.host | default (printf "%s.%s" .Release.Name $host) }}:443
className: "{{ $ingress }}"
additionalServicePorts:
- name: talos-trustd
port: 50001
targetPort: 50001
protocol: TCP
{{- /* Initial render lists only the DNS SANs because the apiserver
Service ClusterIP is not yet allocated when Helm executes the
template. The talos-reconcile post-install/post-upgrade hook
re-patches certSANs with the live ClusterIP once Kamaji emits
the Service, and Kamaji then re-issues the apiserver TLS cert
to cover it. */}}
certSANs:
- {{ printf "%s.%s.svc" .Release.Name .Release.Namespace }}
- {{ printf "%s.%s.svc.%s" .Release.Name .Release.Namespace (include "kubernetes.tenantClusterDomain" .) }}
deployment:
podAdditionalMetadata:
labels:
policy.cozystack.io/allow-to-etcd: "true"
extraContainers:
- name: talos-csr-signer
image: "{{ default (.Files.Get "images/talos-csr-signer.tag" | trim) .Values.images.talosCsrSigner }}"
imagePullPolicy: IfNotPresent
ports:
- name: trustd
containerPort: 50001
protocol: TCP
env:
- name: TALOS_TOKEN
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-talos-secrets
key: token
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 200m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
capabilities:
drop:
- ALL
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: talos-ca
mountPath: /etc/talos-ca
readOnly: true
- name: talos-tls-cert
mountPath: /etc/talos-server-crt
readOnly: true
extraVolumes:
- name: talos-ca
secret:
secretName: {{ .Release.Name }}-talos-ca
- name: talos-tls-cert
secret:
secretName: {{ .Release.Name }}-talos-tls-cert
replicas: {{ .Values.controlPlane.replicas }}
version: {{ include "kubernetes.versionMap" $ }}
---
@@ -406,104 +509,18 @@ metadata:
{{- end }}
{{- end }}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: {{ $.Release.Name }}-{{ $groupName }}
namespace: {{ $.Release.Namespace }}
spec:
template:
spec:
files:
- path: /usr/bin/update-k8s.sh
owner: root:root
permissions: "0755"
content: |
#!/usr/bin/env bash
set -euo pipefail
# Expected to be passed in via preKubeadmCommands
: "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}"
ARCH="$(uname -m)"
case "${ARCH}" in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
esac
# Use your internal mirror here for real-world use.
BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}"
echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..."
curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet
curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm
chmod 0755 /root/kubelet
chmod 0755 /root/kubeadm
if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi
if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi
{{- $sec := lookup "v1" "Secret" $.Release.Namespace (printf "%s-patch-containerd" $.Release.Name) }}
{{- if $sec }}
{{- range $key, $_ := $sec.data }}
- path: /etc/containerd/certs.d/{{ trimSuffix ".toml" $key }}/hosts.toml
contentFrom:
secret:
name: {{ $.Release.Name }}-patch-containerd
key: {{ $key }}
permissions: "0400"
{{- end }}
{{- end }}
diskSetup:
filesystems:
- device: /dev/vdb
filesystem: xfs
label: persistent
partition: "none"
{{/* overwrite must stay false: XFS on /dev/vdb is created on first boot; subsequent reboots of the same VM must reuse the existing filesystem via LABEL=persistent */}}
overwrite: false
mounts:
- ["LABEL=persistent", "/persistent"]
- ["/persistent/kubelet", "/var/lib/kubelet", "none", "bind,nofail"]
- ["/persistent/containerd", "/var/lib/containerd", "none", "bind,nofail"]
preKubeadmCommands:
- KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true
- sed -i 's|root:x:|root::|' /etc/passwd
- systemctl stop containerd.service
- mkdir -p /persistent/kubelet /persistent/containerd /var/lib/kubelet /var/lib/containerd
- mountpoint -q /var/lib/kubelet || mount -o bind /persistent/kubelet /var/lib/kubelet
- mountpoint -q /var/lib/containerd || mount -o bind /persistent/containerd /var/lib/containerd
{{/* containerd 2.x renamed the CRI plugin from io.containerd.grpc.v1.cri to io.containerd.cri.v1.images; both seds run unconditionally and the wrong-version one is a no-op */}}
- sed -i '/\[plugins\.[^]]*grpc\.v1\.cri[^]]*registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml
- sed -i '/\[plugins\.[^]]*io\.containerd\.cri\.v1\.images[^]]*registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml
- systemctl start containerd.service
joinConfiguration:
nodeRegistration:
kubeletExtraArgs:
{{- /*
GPU node-groups: label every node `gpu=on` so HAMi's
hami-device-plugin DaemonSet (nodeSelector gpu=on) schedules
and advertises nvidia.com/gpu. Without the label the plugin
stays at DESIRED=0 and no GPUs are exposed to the tenant.
*/}}
{{- if $group.gpus }}
node-labels: "gpu=on"
{{- end }}
system-reserved: "memory={{ $systemReservedMemory }},cpu={{ $systemReservedCpu }}"
kube-reserved: "memory={{ $kubeReservedMemory }},cpu={{ $kubeReservedCpu }}"
eviction-hard: "memory.available<{{ $evictionHardMemory }},nodefs.available<10%,imagefs.available<15%,nodefs.inodesFree<5%"
eviction-soft: "memory.available<{{ $evictionSoftMemory }},nodefs.available<15%,imagefs.available<20%"
eviction-soft-grace-period: "memory.available=1m30s,nodefs.available=1m30s,imagefs.available=1m30s"
# 256Mi ensures meaningful reclamation per eviction cycle while being
# proportional to the 256Mi-1Gi auto-computed reservation range.
eviction-minimum-reclaim: "memory.available=256Mi"
enforce-node-allocatable: "pods"
ignorePreflightErrors:
- FileExisting-conntrack
discovery:
bootstrapToken:
apiServerEndpoint: {{ $.Release.Name }}.{{ $.Release.Namespace }}.svc:6443
initConfiguration:
skipPhases:
- addon/kube-proxy
{{- /* TalosConfigTemplate is no longer rendered here — the chart cannot
produce a valid Talos worker machineconfig at Helm template time
because three of the inputs (apiserver Service ClusterIP, Talos
CA from cert-manager, Kamaji-issued tenant CA) are created
asynchronously after Helm applies this manifest. The
talos-reconcile-job hook (templates/talos/talos-reconcile-job.yaml)
waits for those inputs and produces the TalosConfigTemplate with
a KamajiControlPlane ownerReference for natural GC. The
MachineDeployment below still references the TCT name; CAPI's
Machine controller blocks on its absence and unblocks the moment
the hook applies it, which happens in the same post-install/
post-upgrade cycle. */}}
---
{{- $context := deepCopy $ }}
{{- $_ := set $context "group" $group }}
@@ -569,8 +586,8 @@ spec:
spec:
bootstrap:
configRef:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
kind: TalosConfigTemplate
name: {{ $.Release.Name }}-{{ $groupName }}
namespace: {{ $.Release.Namespace }}
clusterName: {{ $.Release.Name }}
@@ -588,8 +605,13 @@ metadata:
namespace: {{ $.Release.Namespace }}
spec:
clusterName: {{ $.Release.Name }}
maxUnhealthy: 0
nodeStartupTimeout: 10m
{{- /* MHC spec.maxUnhealthy is an IntOrString. CAPI admission accepts a
bare integer (0, 1, ...) or a percentage string ("0%", "50%"), but
rejects a quoted plain integer ("0"). Quote only when the operator
supplied a percentage; otherwise render the value as a bare int so
the admission webhook is happy with the chart-default "0". */}}
maxUnhealthy: {{ if hasSuffix "%" (toString $.Values.nodeHealthCheck.maxUnhealthy) }}{{ $.Values.nodeHealthCheck.maxUnhealthy | quote }}{{ else }}{{ $.Values.nodeHealthCheck.maxUnhealthy | int }}{{ end }}
nodeStartupTimeout: {{ $.Values.nodeHealthCheck.nodeStartupTimeout }}
selector:
matchLabels:
cluster.x-k8s.io/cluster-name: {{ $.Release.Name }}
@@ -1,13 +0,0 @@
{{- $sourceSecret := lookup "v1" "Secret" "cozy-system" "patch-containerd" }}
{{- if $sourceSecret }}
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-patch-containerd
namespace: {{ .Release.Namespace }}
type: {{ $sourceSecret.type }}
data:
{{- range $key, $value := $sourceSecret.data }}
{{ printf "%s: %s" $key ($value | quote) | indent 2 }}
{{- end }}
{{- end }}
@@ -1,6 +1,14 @@
{{- define "cozystack.defaultCiliumValues" -}}
{{- /* Cilium runs hostNetwork on the Talos worker nodes and resolves the
apiserver via the worker's /etc/hosts, which the talos-reconcile
post-install hook seeds with the Kamaji Service ClusterIP under
<release>.<ns>.svc through machine.network.extraHostEntries. So a
plain DNS host works here without a lookup-and-reuse race: the
cert covers the same DNS name from chart render time, and the
hosts entry is set well before kubelet (and therefore Cilium)
comes up on the node. */}}
cilium:
k8sServiceHost: {{ .Release.Name }}.{{ .Release.Namespace }}.svc
k8sServiceHost: {{ printf "%s.%s.svc" .Release.Name .Release.Namespace }}
k8sServicePort: 6443
routingMode: tunnel
MTU: 1350
@@ -41,7 +41,7 @@ ouroboros:
federations, custom kubelet --cluster-domain, etc.) can
override via
`addons.ouroboros.valuesOverride.ouroboros.controller.clusterDomain`. */}}
clusterDomain: cluster.local
clusterDomain: {{ include "kubernetes.tenantClusterDomain" $ }}
{{- end }}
{{- /* addons.ouroboros.enabled requires addons.ingressNginx.enabled. The
@@ -0,0 +1,152 @@
{{- /*
Job that materialises a long-lived kubeadm-format bootstrap-token Secret
inside the tenant kube-system namespace, so that Talos workers can use
cluster.token from their machineconfig to perform kubelet CSR.
Why a Job and not a kube-native Secret on the management cluster:
bootstrap-token-<id> Secrets must live in the tenant apiserver's
kube-system, which the management cluster can only reach via the
Kamaji-issued admin kubeconfig stored as <release>-admin-kubeconfig.
The Job runs as a Helm post-install/post-upgrade hook so it executes
after the KamajiControlPlane has reconciled and the admin kubeconfig
Secret exists. It is idempotent (kubectl apply) and re-runs harmlessly
on every upgrade.
*/}}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Release.Name }}-bootstrap-token-tenant
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "0"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
---
apiVersion: batch/v1
kind: Job
metadata:
name: {{ .Release.Name }}-bootstrap-token-tenant
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "10"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
spec:
# backoffLimit + restartPolicy: OnFailure gives the Job up to 11 attempts at
# the tenant apiserver with exponential backoff (capped at 6 min between
# restarts). Worst case ~15 min, well inside the chart's HR install timeout
# (release.cozystack.io/helm-install-timeout: 20m on the cozyrds entry) so
# the Job either succeeds within the HR window or fails the chart upgrade
# loudly, instead of letting Helm mark the install "successful" while the
# bootstrap-token Secret never lands in the tenant kube-system.
backoffLimit: 10
ttlSecondsAfterFinished: 600
template:
spec:
restartPolicy: OnFailure
serviceAccountName: {{ .Release.Name }}-bootstrap-token-tenant
containers:
- name: kubectl
image: "{{ default (.Files.Get "images/kubectl.tag" | trim) .Values.images.kubectl }}"
env:
- name: KUBECONFIG
value: /tenant-admin/super-admin.svc
- name: TOKEN_ID
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-talos-secrets
key: bootstrapTokenId
- name: TOKEN_SECRET
valueFrom:
secretKeyRef:
name: {{ .Release.Name }}-talos-secrets
key: bootstrapTokenSecret
volumeMounts:
- name: tenant-admin
mountPath: /tenant-admin
readOnly: true
command:
- sh
- -ec
- |
# The kubectl apply calls below fail with a non-zero exit when
# the tenant apiserver is unreachable, which restarts the pod
# under backoffLimit. No internal retry loop needed — Job-level
# backoff handles the transient unreadiness window after Kamaji
# bootstraps the konnectivity plane.
kubectl version --request-timeout=10s
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
name: bootstrap-token-${TOKEN_ID}
namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
token-id: ${TOKEN_ID}
token-secret: ${TOKEN_SECRET}
description: "Talos worker bootstrap token, managed by cozystack kubernetes chart"
usage-bootstrap-authentication: "true"
usage-bootstrap-signing: "true"
auth-extra-groups: "system:bootstrappers:kubeadm:default-node-token"
EOF
echo "[$(date -u +%H:%M:%S)] bootstrap-token-${TOKEN_ID} ensured"
# Grant system:nodes the ability to watch the kubernetes
# Endpoint/EndpointSlice in the default namespace. Talos
# workers run an internal k8s.EndpointController (apiserver
# failover discovery) that otherwise spams
# "endpointslices forbidden" warnings.
# Note: resourceNames intentionally omitted — Talos's
# informer issues unbounded list/watch and Kubernetes RBAC
# only honours resourceNames on list/watch when the client
# restricts via a field selector, which Talos does not.
# The Role is namespace-scoped to default, where only the
# built-in "kubernetes" endpoint lives.
cat <<'EOF' | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: talos:node-apiserver-endpoint-discovery
namespace: default
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: talos:node-apiserver-endpoint-discovery
namespace: default
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: talos:node-apiserver-endpoint-discovery
subjects:
- kind: Group
apiGroup: rbac.authorization.k8s.io
name: system:nodes
EOF
# Drop the previous ClusterRole/Binding incarnation if it
# exists from earlier deploys of this chart.
kubectl delete clusterrole talos:node-apiserver-endpoint-discovery --ignore-not-found
kubectl delete clusterrolebinding talos:node-apiserver-endpoint-discovery --ignore-not-found
echo "[$(date -u +%H:%M:%S)] talos:node-apiserver-endpoint-discovery RBAC ensured"
volumes:
- name: tenant-admin
secret:
secretName: {{ .Release.Name }}-admin-kubeconfig
@@ -0,0 +1,495 @@
{{- /*
Post-install/post-upgrade Helm hook that resolves the chart's
lookup-and-reuse race for TalosConfigTemplate and
KamajiControlPlane.spec.network.certSANs.
Background. The Talos worker machineconfig and the apiserver Service
ClusterIP-SAN both need values that only exist AFTER Helm has applied
the first render: the Service is emitted by the Kamaji controller in
reaction to the KamajiControlPlane CR, the Talos-CA TLS Secret is
issued by cert-manager from the Talos PKI Certificate CR, and the
Kubernetes-CA Secret is generated by Kamaji as part of its tenant
control-plane bootstrap. A Helm template that uses `lookup` to read
any of these into the rendered manifest fails the race on every fresh
install — the lookup returns nil, the gated TalosConfigTemplate is
skipped, and Helm's release-storage three-way merge concludes the
release is in-sync on every subsequent reconcile because no chart
value changed. The result is a tenant whose KamajiControlPlane is
Ready but whose CAPI MachineDeployment is stuck on
`cannot create a new MachineSet when templates do not exist` forever.
This Job runs once Helm has applied everything else in the release,
blocks until the four dependencies are observably present, then
produces both objects via `kubectl apply` (idempotent). The
TalosConfigTemplate carries an ownerReference to the KamajiControlPlane
so Kubernetes garbage-collects it through the standard owner-reference
cascade when the chart's release is uninstalled — Helm itself does not
need to track the resource, and a `pre-delete` hook is not required.
The certSANs patch is a merge against the KCP `spec.network.certSANs`
field; the Kamaji controller picks up the change and re-issues the
apiserver TLS cert with the live ClusterIP in its SAN list.
Hook weight 5 places the Job after Helm has emitted the
KamajiControlPlane / Talos PKI Certificate / talos-secrets but before
the bootstrap-token-tenant Job (weight 10), so workers see a
reachable apiserver, a SAN-correct TLS cert, and a populated
bootstrap-token Secret in the tenant kube-system the first time
they boot.
*/}}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Release.Name }}-talos-reconcile
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ .Release.Name }}-talos-reconcile
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
rules:
- apiGroups: [""]
resources: ["services"]
resourceNames: ["{{ .Release.Name }}"]
verbs: ["get"]
- apiGroups: [""]
resources: ["secrets"]
resourceNames:
- "{{ .Release.Name }}-talos-secrets"
- "{{ .Release.Name }}-talos-ca"
- "{{ .Release.Name }}-ca"
verbs: ["get"]
- apiGroups: ["controlplane.cluster.x-k8s.io"]
resources: ["kamajicontrolplanes"]
resourceNames: ["{{ .Release.Name }}"]
verbs: ["get", "patch"]
- apiGroups: ["bootstrap.cluster.x-k8s.io"]
resources: ["talosconfigtemplates"]
verbs: ["get", "create", "patch", "update"]
---
{{- /* The tenant-root chart installs a default-deny egress policy on every
pod in the tenant namespace; without an explicit allow, this hook Job
cannot reach the management-cluster apiserver to read the Service
ClusterIP / Secrets / KamajiControlPlane it has to reconcile. Cilium
policies are additive, so this rule layers on top of the tenant
default-deny without weakening it for anything else. Endpoint
selector is scoped to pods carrying the hook label only. */}}
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: {{ .Release.Name }}-talos-reconcile
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
spec:
endpointSelector:
matchLabels:
cozystack.io/talos-reconcile: "{{ .Release.Name }}"
egress:
- toEntities:
- kube-apiserver
- toEndpoints:
- matchLabels:
"k8s:io.kubernetes.pod.namespace": kube-system
"k8s:k8s-app": kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ .Release.Name }}-talos-reconcile
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ .Release.Name }}-talos-reconcile
subjects:
- kind: ServiceAccount
name: {{ .Release.Name }}-talos-reconcile
namespace: {{ .Release.Namespace }}
---
{{- /* Cluster-scoped read on the management CoreDNS Service so the hook
can look up its ClusterIP at runtime and inject it into the worker
machineconfig as a host-level nameserver. Tenant workers run as
KubeVirt VMs inside the management pod network; tenant in-cluster
pods still use kubelet --cluster-dns (tenant CoreDNS) via
dnsPolicy: ClusterFirst, but the worker host itself (kubelet's
mount syscall path, image pulls) consults /etc/resolv.conf which
Talos populates from machine.network.nameservers. Without
management CoreDNS as host resolver, mounting NFS from
kubevirt-csi (linstor-csi-nfs.cozy-linstor.svc) fails with
NXDOMAIN.
Ubuntu+kubeadm workers in main inherit pod DNS via cloud-init
and got management CoreDNS for free; Talos reads only what is
in machineconfig, so the chart has to plumb it explicitly.
resourceNames pins the grant to the single well-known Service
name kube-dns — cluster-wide get on services scoped to that
single name is functionally a single-resource permission. */}}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
rules:
- apiGroups: [""]
resources: ["services"]
resourceNames: ["kube-dns"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns
subjects:
- kind: ServiceAccount
name: {{ .Release.Name }}-talos-reconcile
namespace: {{ .Release.Namespace }}
---
{{- $kubeletVersion := include "kubernetes.versionMap" $ | trim }}
{{- $talosVersion := $.Values.talos.version }}
{{- $podCIDR := "10.243.0.0/16" }}
{{- $serviceCIDR := "10.95.0.0/16" }}
{{- $dnsDomain := include "kubernetes.tenantClusterDomain" $ }}
{{- /* Management cluster DNS domain. Tenant Kubernetes workers run as
KubeVirt VMs inside the management cluster's pod network and need
to resolve management-cluster service short names ("<svc>.<ns>.svc")
the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The
kubevirt-csi NFS mount path is the load-bearing case: kubelet on
the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc`
(no domain suffix), and without searchDomains in /etc/resolv.conf
the host resolver hands that name to management CoreDNS verbatim
and gets NXDOMAIN.
Distinct from tenantClusterDomain ("cluster.local"), which is the
*tenant* kubelet --cluster-dns zone. This value is the
*management* cluster-domain, sourced from .Values._cluster
(populated by cozystack-controller from the platform config). */}}
{{- $mgmtClusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }}
{{- range $groupName, $group := .Values.nodeGroups }}
{{- /* Mirror the kubelet-reservation computation from cluster.yaml so the
worker machineconfig the hook applies carries the same numbers the
MachineDeployment validator was happy with. Auto-computed
system/kube reserved memory = 5% of effective memory clamped to
[256Mi, 1Gi]; auto-computed system/kube reserved CPU = 5% of
effective CPU clamped to [50m, 500m]. instanceType lookup is
optional — operators that override resources.cpu / resources.memory
on the nodeGroup get those, otherwise the chart falls back to the
same static-defaults pair the cluster.yaml validator uses on a
no-instanceType setup. */}}
{{- $instanceType := dict }}
{{- if $group.instanceType }}
{{- $instanceType = lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType }}
{{- $instanceType = $instanceType | default dict }}
{{- end }}
{{- $effectiveMemory := "" }}
{{- if and $group.resources $group.resources.memory }}
{{- $effectiveMemory = $group.resources.memory | toString }}
{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }}
{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }}
{{- end }}
{{- $effectiveCpu := "" }}
{{- if and $group.resources $group.resources.cpu }}
{{- $effectiveCpu = $group.resources.cpu | toString }}
{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }}
{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }}
{{- end }}
{{- $autoReservedMi := 256 }}
{{- if $effectiveMemory }}
{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }}
{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }}
{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }}
{{- end }}
{{- $autoReservedMillicores := 50 }}
{{- if $effectiveCpu }}
{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }}
{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }}
{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }}
{{- end }}
{{- $kubeletOverride := $group.kubelet | default dict }}
{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }}
{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }}
{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }}
{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }}
{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }}
{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ $.Release.Name }}-talos-reconcile-{{ $groupName }}
namespace: {{ $.Release.Namespace }}
labels:
{{- include "kubernetes.labels" $ | nindent 4 }}
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "5"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
spec:
backoffLimit: 5
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
cozystack.io/talos-reconcile: {{ $.Release.Name | quote }}
spec:
restartPolicy: OnFailure
serviceAccountName: {{ $.Release.Name }}-talos-reconcile
containers:
- name: kubectl
image: "{{ default ($.Files.Get "images/kubectl.tag" | trim) $.Values.images.kubectl }}"
env:
- name: RELEASE
value: {{ $.Release.Name | quote }}
- name: NS
value: {{ $.Release.Namespace | quote }}
- name: GROUP_NAME
value: {{ $groupName | quote }}
- name: TALOS_TOKEN
valueFrom:
secretKeyRef:
name: {{ $.Release.Name }}-talos-secrets
key: token
- name: CLUSTER_ID
valueFrom:
secretKeyRef:
name: {{ $.Release.Name }}-talos-secrets
key: clusterId
- name: CLUSTER_SECRET
valueFrom:
secretKeyRef:
name: {{ $.Release.Name }}-talos-secrets
key: clusterSecret
- name: BOOTSTRAP_TOKEN
valueFrom:
secretKeyRef:
name: {{ $.Release.Name }}-talos-secrets
key: bootstrapToken
command:
- sh
- -ec
- |
echo "[$(date -u +%H:%M:%S)] talos-reconcile/${GROUP_NAME}: waiting for Service ClusterIP, Talos CA, K8s CA, management CoreDNS"
for i in $(seq 1 120); do
SVC_IP=$(kubectl -n "$NS" get svc "$RELEASE" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)
TALOS_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-talos-ca" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)
K8S_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true)
COREDNS_IP=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)
if [ -n "$SVC_IP" ] && [ -n "$TALOS_CA_B64" ] && [ -n "$K8S_CA_B64" ] && [ -n "$COREDNS_IP" ]; then
break
fi
sleep 5
done
if [ -z "$SVC_IP" ] || [ -z "$TALOS_CA_B64" ] || [ -z "$K8S_CA_B64" ] || [ -z "$COREDNS_IP" ]; then
echo "[$(date -u +%H:%M:%S)] timed out waiting for lookup inputs (svc='$SVC_IP' talos-ca=$([ -n "$TALOS_CA_B64" ] && echo ok) k8s-ca=$([ -n "$K8S_CA_B64" ] && echo ok) coredns='$COREDNS_IP')"
exit 1
fi
echo "[$(date -u +%H:%M:%S)] inputs ready (svc=${SVC_IP} coredns=${COREDNS_IP})"
# Patch KamajiControlPlane.spec.network.certSANs so the apiserver
# TLS cert covers the live Service ClusterIP that Talos workers
# dial. JSON-merge-patch is array-replace semantics, which is what
# we want here — Kamaji owns the full SAN list.
kubectl -n "$NS" patch kamajicontrolplane "$RELEASE" --type=merge --patch "$(
cat <<JSON
{"spec":{"network":{"certSANs":["${RELEASE}.${NS}.svc","${RELEASE}.${NS}.svc.{{ $dnsDomain }}","${SVC_IP}"]}}}
JSON
)"
# Talos workers dial the apiserver Service via the DNS name
# (${RELEASE}.${NS}.svc) instead of the live ClusterIP, with
# machine.network.extraHostEntries injecting the IP into the
# worker's /etc/hosts. This avoids a TLS race against the
# csr-signer sidecar: the Talos PKI Certificate is issued
# with DNS SANs at chart render time and never needs to be
# patched after the Service ClusterIP becomes known. The
# apiserver Kubernetes TLS cert (handled by KCP certSANs above)
# auto-reloads on Kamaji reissue, but the csr-signer container
# loads its TLS cert once at startup and does not watch the
# file — so any in-place cert rotation leaves it serving the
# original (no-IP-SAN) cert until the Kamaji pod restarts.
# Routing through DNS sidesteps the whole problem.
KCP_UID=$(kubectl -n "$NS" get kamajicontrolplane "$RELEASE" -o jsonpath='{.metadata.uid}')
# Apply the TalosConfigTemplate. ownerReference back to the KCP
# makes the standard Kubernetes garbage collector clean it up on
# release uninstall; Helm itself does not track this resource.
cat <<EOF | kubectl apply -f -
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
kind: TalosConfigTemplate
metadata:
name: ${RELEASE}-${GROUP_NAME}
namespace: ${NS}
ownerReferences:
- apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
kind: KamajiControlPlane
name: ${RELEASE}
uid: ${KCP_UID}
controller: true
blockOwnerDeletion: true
spec:
template:
spec:
generateType: none
talosVersion: "{{ $talosVersion }}"
data: |
version: v1alpha1
persist: true
machine:
type: worker
token: ${TALOS_TOKEN}
network:
# Management CoreDNS as the worker's host
# resolver, plus management cluster search
# domains so partial names like
# "linstor-csi-nfs.cozy-linstor.svc" resolve
# the same way they do on Ubuntu+kubeadm
# workers in main (which got these via
# cloud-init/DHCP). Without searchDomains the
# kubevirt-csi NFS mount fails NXDOMAIN
# because mount.nfs hands the partial name
# to the resolver verbatim and management
# CoreDNS only knows full FQDNs.
#
# Tenant in-cluster pods are not affected:
# dnsPolicy: ClusterFirst routes them through
# kubelet --cluster-dns (tenant CoreDNS).
# This block only configures host-side
# resolution (kubelet mount syscalls, image
# pulls, pods opted out of cluster DNS).
nameservers:
- ${COREDNS_IP}
searchDomains:
- svc.{{ $mgmtClusterDomain }}
- {{ $mgmtClusterDomain }}
{{- if ne $mgmtClusterDomain "cluster.local" }}
- svc.cluster.local
- cluster.local
{{- end }}
extraHostEntries:
- ip: ${SVC_IP}
aliases:
- ${RELEASE}.${NS}.svc
- ${RELEASE}.${NS}.svc.{{ $dnsDomain }}
ca:
crt: ${TALOS_CA_B64}
{{- if $group.gpus }}
# GPU node-groups: label every node `gpu=on` so
# HAMi's hami-device-plugin DaemonSet (nodeSelector
# gpu=on) schedules and advertises nvidia.com/gpu.
# Without the label the plugin stays at DESIRED=0
# and no GPUs are exposed to the tenant. Mirrors
# main's kubeadm config (kubeletExtraArgs node-
# labels: "gpu=on") so HAMi behaviour is identical
# before and after the Talos worker rollover.
nodeLabels:
gpu: "on"
{{- end }}
kubelet:
image: ghcr.io/siderolabs/kubelet:{{ $kubeletVersion }}
extraConfig:
systemReserved:
cpu: "{{ $systemReservedCpu }}"
memory: "{{ $systemReservedMemory }}"
kubeReserved:
cpu: "{{ $kubeReservedCpu }}"
memory: "{{ $kubeReservedMemory }}"
evictionHard:
memory.available: "{{ $evictionHardMemory }}"
nodefs.available: "10%"
imagefs.available: "15%"
nodefs.inodesFree: "5%"
evictionSoft:
memory.available: "{{ $evictionSoftMemory }}"
nodefs.available: "15%"
imagefs.available: "20%"
evictionSoftGracePeriod:
memory.available: "1m30s"
nodefs.available: "1m30s"
imagefs.available: "1m30s"
evictionMinimumReclaim:
memory.available: "256Mi"
install:
disk: /dev/vda
image: factory.talos.dev/installer/{{ $.Values.talos.schematicID }}:{{ $talosVersion }}
wipe: false
features:
rbac: true
kubePrism:
enabled: false
cluster:
id: ${CLUSTER_ID}
secret: ${CLUSTER_SECRET}
controlPlane:
endpoint: https://${RELEASE}.${NS}.svc:6443
clusterName: ${RELEASE}
network:
dnsDomain: {{ $dnsDomain }}
podSubnets:
- {{ $podCIDR }}
serviceSubnets:
- {{ $serviceCIDR }}
token: ${BOOTSTRAP_TOKEN}
ca:
crt: ${K8S_CA_B64}
discovery:
enabled: true
registries:
kubernetes:
disabled: true
service:
disabled: true
EOF
echo "[$(date -u +%H:%M:%S)] TalosConfigTemplate ${RELEASE}-${GROUP_NAME} applied"
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
cpu: 100m
memory: 256Mi
{{- end }}
+54 -3
View File
@@ -45,7 +45,7 @@
],
"properties": {
"diskSize": {
"description": "Persistent disk size for kubelet and containerd data.",
"description": "System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.",
"default": "20Gi",
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
"anyOf": [
@@ -185,8 +185,7 @@
"v1.34",
"v1.33",
"v1.32",
"v1.31",
"v1.30"
"v1.31"
]
},
"host": {
@@ -898,12 +897,64 @@
"type": "object",
"default": {},
"properties": {
"kubectl": {
"description": "Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.",
"type": "string",
"default": ""
},
"talosCsrSigner": {
"description": "Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.",
"type": "string",
"default": ""
},
"waitForKubeconfig": {
"description": "Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.",
"type": "string",
"default": ""
}
}
},
"talos": {
"description": "Talos worker image configuration.",
"type": "object",
"default": {},
"required": [
"schematicID",
"version"
],
"properties": {
"schematicID": {
"description": "Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.",
"type": "string",
"default": "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
},
"version": {
"description": "Talos release used for worker OS image and installer. Must satisfy the chart's Talos\u003c-\u003eKubernetes support matrix against the chosen `version`.",
"type": "string",
"default": "v1.13.0"
}
}
},
"nodeHealthCheck": {
"description": "MachineHealthCheck tuning for worker node groups.",
"type": "object",
"default": {},
"required": [
"maxUnhealthy",
"nodeStartupTimeout"
],
"properties": {
"maxUnhealthy": {
"description": "Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default \"50%\" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to \"0%\" once the fleet is stable on Talos workers.",
"type": "string",
"default": "50%"
},
"nodeStartupTimeout": {
"description": "Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).",
"type": "string",
"default": "10m"
}
}
}
}
}
+31 -2
View File
@@ -81,7 +81,7 @@ storageClass: replicated
## @field {int} maxReplicas=10 - Maximum number of replicas.
## @field {string} instanceType="u1.medium" - Virtual machine instance type.
## @x-cozystack-options {source: instancetype}
## @field {quantity} diskSize="20Gi" - Persistent disk size for kubelet and containerd data.
## @field {quantity} diskSize="20Gi" - System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.
## @field {string} [storageClass] - StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.
## @x-cozystack-options {source: storageclass}
## @field {[]string} roles - List of node roles.
@@ -110,7 +110,6 @@ nodeGroups:
## @value v1.33
## @value v1.32
## @value v1.31
## @value v1.30
## @param {Version} version - Kubernetes major.minor version to deploy
version: "v1.35"
@@ -273,7 +272,37 @@ controlPlane:
## @typedef {struct} Images - Optional image overrides for chart-internal helpers.
## @field {string} [waitForKubeconfig] - Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
## @field {string} [kubectl] - Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.
## @field {string} [talosCsrSigner] - Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.
## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries.
images:
waitForKubeconfig: ""
kubectl: ""
talosCsrSigner: ""
##
## @section Talos Worker Image
##
## @typedef {struct} Talos - Talos worker image configuration.
## @field {string} version - Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.
## @field {string} schematicID - Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.
## @param {Talos} talos - Talos worker image configuration.
talos:
version: "v1.13.0"
schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
##
## @section Node Health Check
##
## @typedef {struct} NodeHealthCheck - MachineHealthCheck configuration for worker node groups.
## @field {string} maxUnhealthy="50%" - Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers.
## @field {string} nodeStartupTimeout="10m" - Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).
## @param {NodeHealthCheck} nodeHealthCheck - MachineHealthCheck tuning for worker node groups.
nodeHealthCheck:
maxUnhealthy: "50%"
nodeStartupTimeout: "10m"
@@ -15,14 +15,24 @@ metadata:
# deadline chosen to stay strictly below this value so the pod's
# CrashLoopBackOff surfaces before flux remediation fires. If this
# annotation is raised, update that init deadline correspondingly.
release.cozystack.io/helm-install-timeout: "15m"
release.cozystack.io/helm-install-timeout: "20m"
# The kubernetes chart emits in-tenant addon HelmReleases (cilium, coredns,
# csi, ingress-nginx, metrics-server, ouroboros) and ships a talos-reconcile
# post-install hook that produces the TalosConfigTemplate the worker
# MachineSets clone from. Without DisableWait the helm-controller waits for
# the addon HelmReleases to be Ready before running the hook, but those
# cannot be Ready until worker nodes exist, and the workers cannot be
# created without the TalosConfigTemplate the hook is supposed to produce.
# Skip the readiness wait so the hook fires and the chicken-and-egg
# resolves.
release.cozystack.io/helm-install-disable-wait: "true"
spec:
application:
kind: Kubernetes
singular: kubernetes
plural: kuberneteses
openAPISchema: |-
{"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.Line truncated
{"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ciliuLine truncated
release:
prefix: kubernetes-
labels:
@@ -40,7 +50,7 @@ spec:
tags:
- compute
icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MLine truncated
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "diskSize"], ["spec", "nodeGroups", "md0", "storageClass"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "nodeGroups", "md0", "kubelet"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]]
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "diskSize"], ["spec", "nodeGroups", "md0", "storageClass"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "nodeGroups", "md0", "kubelet"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"], ["spec", "images", "kubectl"], ["spec", "images", "talosCsrSigner"], ["spec", "talos"], ["spec", "talos", "version"], ["spec", "talos", "schematicID"], ["spec", "nodeHealthCheck"], ["spec", "nodeHealthCheck", "maxUnhealthy"], ["spec", "nodeHealthCheck", "nodeStartupTimeout"]]
secrets:
exclude: []
include: