mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-03 10:37:21 +09:00
feat(kubernetes): bootstrap tenant workers with Talos via CABPT
Switch tenant worker bootstrap from Ubuntu+kubeadm to Talos. MachineDeployment.bootstrap moves from KubeadmConfigTemplate to a TalosConfigTemplate produced by a post-install/upgrade reconcile hook, which also materialises the kubeadm-format bootstrap-token Secret in the tenant kube-system. Workers boot from the Talos openstack image via a CDI DataVolume; the system disk is virtio-blk with logical=512/physical=4096 so 4Ki-native backends (LINSTOR/DRBD) and SeaBIOS both work; the separate kubelet disk is dropped. Pod/service CIDRs, dnsDomain, controlPlane endpoint and cilium k8sServiceHost are pinned to the tenant ranges / Kamaji apiserver ClusterIP to survive the DNS chicken-and-egg at bootstrap. Adds a Talos<->Kubernetes support-matrix fail() guard and drops v1.30. Removes the dead ubuntu-container-disk build chain and the orphaned copy-patch-containerd template. Co-authored-by: IvanHunters <xorokhotnikov@gmail.com> Co-authored-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com> Assisted-By: Claude <noreply@anthropic.com> Signed-off-by: Andrei Kvapil <kvapss@gmail.com> Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
1 parent
ccf6d9dbfe
commit
39cdc88661
23 files changed
+1049
-261
No files matched your search
@@ -40,6 +40,12 @@ type ConfigSpec struct {
|
||||
// Optional image overrides for air-gapped or rate-limited registries.
|
||||
// +kubebuilder:default:={}
|
||||
Images Images `json:"images"`
|
||||
// Talos worker image configuration.
|
||||
// +kubebuilder:default:={}
|
||||
Talos Talos `json:"talos"`
|
||||
// MachineHealthCheck tuning for worker node groups.
|
||||
// +kubebuilder:default:={}
|
||||
NodeHealthCheck NodeHealthCheck `json:"nodeHealthCheck"`
|
||||
}
|
||||
|
||||
type APIServer struct {
|
||||
@@ -177,6 +183,12 @@ type HAMiAddon struct {
|
||||
}
|
||||
|
||||
type Images struct {
|
||||
// Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.
|
||||
// +kubebuilder:default:=""
|
||||
Kubectl string `json:"kubectl,omitempty"`
|
||||
// Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.
|
||||
// +kubebuilder:default:=""
|
||||
TalosCsrSigner string `json:"talosCsrSigner,omitempty"`
|
||||
// Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
|
||||
// +kubebuilder:default:=""
|
||||
WaitForKubeconfig string `json:"waitForKubeconfig,omitempty"`
|
||||
@@ -239,7 +251,7 @@ type MonitoringAgentsAddon struct {
|
||||
}
|
||||
|
||||
type NodeGroup struct {
|
||||
// Persistent disk size for kubelet and containerd data.
|
||||
// System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.
|
||||
// +kubebuilder:default:="20Gi"
|
||||
DiskSize resource.Quantity `json:"diskSize"`
|
||||
// List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
|
||||
@@ -263,6 +275,15 @@ type NodeGroup struct {
|
||||
StorageClass string `json:"storageClass,omitempty"`
|
||||
}
|
||||
|
||||
type NodeHealthCheck struct {
|
||||
// Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers.
|
||||
// +kubebuilder:default:="50%"
|
||||
MaxUnhealthy string `json:"maxUnhealthy"`
|
||||
// Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).
|
||||
// +kubebuilder:default:="10m"
|
||||
NodeStartupTimeout string `json:"nodeStartupTimeout"`
|
||||
}
|
||||
|
||||
type OuroborosAddon struct {
|
||||
// Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.
|
||||
// +kubebuilder:default:=false
|
||||
@@ -288,6 +309,15 @@ type Scheduler struct {
|
||||
ResourcesPreset ResourcesPreset `json:"resourcesPreset"`
|
||||
}
|
||||
|
||||
type Talos struct {
|
||||
// Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.
|
||||
// +kubebuilder:default:="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
SchematicID string `json:"schematicID"`
|
||||
// Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.
|
||||
// +kubebuilder:default:="v1.13.0"
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
type VeleroAddon struct {
|
||||
// Enable Velero.
|
||||
// +kubebuilder:default:=false
|
||||
@@ -309,5 +339,5 @@ type IngressNginxExposeMethod string
|
||||
// +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge"
|
||||
type ResourcesPreset string
|
||||
|
||||
// +kubebuilder:validation:Enum="v1.35";"v1.34";"v1.33";"v1.32";"v1.31";"v1.30"
|
||||
// +kubebuilder:validation:Enum="v1.35";"v1.34";"v1.33";"v1.32";"v1.31"
|
||||
type Version string
|
||||
@@ -138,6 +138,8 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
|
||||
in.Addons.DeepCopyInto(&out.Addons)
|
||||
in.ControlPlane.DeepCopyInto(&out.ControlPlane)
|
||||
out.Images = in.Images
|
||||
out.Talos = in.Talos
|
||||
out.NodeHealthCheck = in.NodeHealthCheck
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec.
|
||||
@@ -406,6 +408,21 @@ func (in *NodeGroup) DeepCopy() *NodeGroup {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NodeHealthCheck) DeepCopyInto(out *NodeHealthCheck) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeHealthCheck.
|
||||
func (in *NodeHealthCheck) DeepCopy() *NodeHealthCheck {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(NodeHealthCheck)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *OuroborosAddon) DeepCopyInto(out *OuroborosAddon) {
|
||||
*out = *in
|
||||
@@ -455,6 +472,21 @@ func (in *Scheduler) DeepCopy() *Scheduler {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Talos) DeepCopyInto(out *Talos) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Talos.
|
||||
func (in *Talos) DeepCopy() *Talos {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(Talos)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *VeleroAddon) DeepCopyInto(out *VeleroAddon) {
|
||||
*out = *in
|
||||
|
||||
@@ -8,7 +8,6 @@ include ../../../hack/package.mk
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
bash tests/containerd-config-path-test.sh
|
||||
|
||||
generate:
|
||||
cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go
|
||||
@@ -18,21 +17,17 @@ update:
|
||||
hack/update-versions.sh
|
||||
make generate
|
||||
|
||||
image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler
|
||||
image: image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler image-talos-csr-signer
|
||||
|
||||
image-ubuntu-container-disk:
|
||||
$(foreach ver,$(KUBERNETES_VERSIONS), \
|
||||
docker buildx build images/ubuntu-container-disk \
|
||||
--build-arg KUBERNETES_VERSION=$(ver) \
|
||||
--tag $(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG) \
|
||||
$(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$(ver)) \
|
||||
$(call cache-args,ubuntu-container-disk,$(ver)-buildcache) \
|
||||
--metadata-file images/ubuntu-container-disk-$(ver).json \
|
||||
$(BUILDX_ARGS) && \
|
||||
echo "$(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$(ver).json -o json -r)" \
|
||||
> images/ubuntu-container-disk-$(ver).tag && \
|
||||
rm -f images/ubuntu-container-disk-$(ver).json; \
|
||||
)
|
||||
image-talos-csr-signer:
|
||||
docker buildx build images/talos-csr-signer \
|
||||
$(call image-tags,talos-csr-signer,$(KUBERNETES_PKG_TAG)) \
|
||||
$(call cache-args,talos-csr-signer) \
|
||||
--metadata-file images/talos-csr-signer.json \
|
||||
$(BUILDX_ARGS)
|
||||
echo "$(REGISTRY)/talos-csr-signer:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/talos-csr-signer.json -o json -r)" \
|
||||
> images/talos-csr-signer.tag
|
||||
rm -f images/talos-csr-signer.json
|
||||
|
||||
image-kubevirt-cloud-provider:
|
||||
docker buildx build images/kubevirt-cloud-provider \
|
||||
|
||||
@@ -85,7 +85,9 @@ See the reference for components utilized in this service:
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize` to better reflect its purpose (persistent disk for kubelet and containerd data). Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs will be automatically rolling-updated via CAPI when the new values are applied. State persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh PVC.
|
||||
- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize`. Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs are rolling-updated via CAPI when the new values are applied. With the Talos worker rollover in this release the field now sizes the **single system disk** (Talos OS image streamed from `factory.talos.dev`, kubelet state, containerd image cache, local-path PVCs) — the pre-Talos `disk-kubelet` PVC layout was removed. State on that single disk persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh disk.
|
||||
|
||||
- **Air-gapped tenant workers temporarily unsupported (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos image fetched over HTTPS from `factory.talos.dev` and a `factory.talos.dev/installer/...` installer reference, neither of which is currently overridable. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig and was removed in this release. Tenants behind a strict-egress firewall or relying on `registries.mirrors` for tenant workers will be unable to create new workers until Phase 2 restores this via `machine.registries.mirrors` knobs and an installable/imageBase override. Tracked as a follow-up to this Phase 1 PR — file an issue if you depend on this and it is not yet landed.
|
||||
|
||||
> The top-level `storageClass` field is annotated as immutable in the chart schema — see [`docs/storage-immutability.md`](../../../docs/storage-immutability.md) for the contract and which consumers enforce it. The per-node-group `nodeGroups[name].storageClass` field is intentionally **not** annotated immutable: it is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.
|
||||
|
||||
@@ -106,7 +108,7 @@ See the reference for components utilized in this service:
|
||||
| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` |
|
||||
| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` |
|
||||
| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` |
|
||||
| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` |
|
||||
| `nodeGroups[name].diskSize` | System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact. | `quantity` | `20Gi` |
|
||||
| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` |
|
||||
| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` |
|
||||
| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` |
|
||||
@@ -168,33 +170,53 @@ See the reference for components utilized in this service:
|
||||
|
||||
### Kubernetes Control Plane Configuration
|
||||
|
||||
| Name | Description | Type | Value |
|
||||
| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ----------- |
|
||||
| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` |
|
||||
| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` |
|
||||
| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` |
|
||||
| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` |
|
||||
| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` |
|
||||
| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` |
|
||||
| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` |
|
||||
| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
|
||||
| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` |
|
||||
| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` |
|
||||
| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
|
||||
| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` |
|
||||
| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` |
|
||||
| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` |
|
||||
| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
|
||||
| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` |
|
||||
| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` |
|
||||
| Name | Description | Type | Value |
|
||||
| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ---------- | ----------- |
|
||||
| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` |
|
||||
| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` |
|
||||
| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` |
|
||||
| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` |
|
||||
| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` |
|
||||
| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` |
|
||||
| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` |
|
||||
| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
|
||||
| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` |
|
||||
| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` |
|
||||
| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
|
||||
| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` |
|
||||
| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` |
|
||||
| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` |
|
||||
| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` |
|
||||
| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` |
|
||||
| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` |
|
||||
| `images.kubectl` | Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag. | `string` | `""` |
|
||||
| `images.talosCsrSigner` | Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag. | `string` | `""` |
|
||||
|
||||
|
||||
### Talos Worker Image
|
||||
|
||||
| Name | Description | Type | Value |
|
||||
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------ |
|
||||
| `talos` | Talos worker image configuration. | `object` | `{}` |
|
||||
| `talos.version` | Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. | `string` | `v1.13.0` |
|
||||
| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` |
|
||||
|
||||
|
||||
### Node Health Check
|
||||
|
||||
| Name | Description | Type | Value |
|
||||
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ----- |
|
||||
| `nodeHealthCheck` | MachineHealthCheck tuning for worker node groups. | `object` | `{}` |
|
||||
| `nodeHealthCheck.maxUnhealthy` | Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers. | `string` | `50%` |
|
||||
| `nodeHealthCheck.nodeStartupTimeout` | Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). | `string` | `10m` |
|
||||
|
||||
|
||||
## Parameter examples and reference
|
||||
@@ -213,9 +235,15 @@ resources:
|
||||
`resourcesPreset` sets named CPU and memory configurations for each replica.
|
||||
This setting is ignored if the corresponding `resources` value is set.
|
||||
|
||||
Presets follow a cloud-style `<series>.<size>` naming convention. Five series cover the full CPU-to-memory ratio range (`t1` 1:0.5, `c1` 1:1, `s1` 1:2, `u1` 1:4, `m1` 1:8) and each series ships eight sizes (`nano` through `4xlarge`). The legacy flat names (`nano`, `micro`, `small`, `medium`, `large`, `xlarge`, `2xlarge`) remain accepted as deprecated aliases of their 1:1 instance-type equivalents.
|
||||
|
||||
See [`docs/operations/resource-presets.md`](../../../docs/operations/resource-presets.md) for the full size matrix and the legacy-to-instance-type mapping.
|
||||
| Preset name | CPU | memory |
|
||||
|-------------|--------|---------|
|
||||
| `nano` | `250m` | `128Mi` |
|
||||
| `micro` | `500m` | `256Mi` |
|
||||
| `small` | `1` | `512Mi` |
|
||||
| `medium` | `1` | `1Gi` |
|
||||
| `large` | `2` | `2Gi` |
|
||||
| `xlarge` | `4` | `4Gi` |
|
||||
| `2xlarge` | `8` | `8Gi` |
|
||||
|
||||
### instanceType Resources
|
||||
|
||||
|
||||
@@ -3,4 +3,3 @@
|
||||
"v1.33": "v1.33.8"
|
||||
"v1.32": "v1.32.12"
|
||||
"v1.31": "v1.31.14"
|
||||
"v1.30": "v1.30.14"
|
||||
@@ -0,0 +1 @@
|
||||
docker.io/alpine/k8s:1.33.4@sha256:b0523f0a244ddc4c8e055aa335c040d3d78b3ead5528f4544395f7f9f69c7b68
|
||||
@@ -1 +0,0 @@
|
||||
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.0@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd
|
||||
@@ -1 +0,0 @@
|
||||
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.0@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70
|
||||
@@ -1 +0,0 @@
|
||||
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.0@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9
|
||||
@@ -1 +0,0 @@
|
||||
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.0@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184
|
||||
@@ -1 +0,0 @@
|
||||
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.0@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574
|
||||
@@ -1 +0,0 @@
|
||||
ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.0@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2
|
||||
@@ -1,60 +0,0 @@
|
||||
# TODO: Here we use ubuntu:22.04, as guestfish has some network issues running in ubuntu:24.04
|
||||
FROM ubuntu:22.04 AS guestfish
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update \
|
||||
&& apt-get -y install \
|
||||
libguestfs-tools \
|
||||
linux-image-generic \
|
||||
wget \
|
||||
make \
|
||||
bash-completion
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
FROM guestfish AS builder
|
||||
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
|
||||
# noble is a code name for the Ubuntu 24.04 LTS release
|
||||
RUN wget -O image.img https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-${TARGETARCH}.img --show-progress --output-file /dev/stdout --progress=dot:giga 2>/dev/null
|
||||
|
||||
ARG KUBERNETES_VERSION
|
||||
|
||||
RUN qemu-img resize image.img 5G \
|
||||
&& eval "$(guestfish --listen --network)" \
|
||||
&& guestfish --remote add-drive image.img \
|
||||
&& guestfish --remote run \
|
||||
&& guestfish --remote mount /dev/sda1 / \
|
||||
&& guestfish --remote command "growpart /dev/sda 1 --verbose" \
|
||||
&& guestfish --remote command "resize2fs /dev/sda1" \
|
||||
# docker repo
|
||||
&& guestfish --remote sh "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg" \
|
||||
&& guestfish --remote sh 'echo "deb [signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list' \
|
||||
# kubernetes repo
|
||||
&& guestfish --remote sh "curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg" \
|
||||
&& guestfish --remote sh "echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list" \
|
||||
&& guestfish --remote command "apt-get check -q" \
|
||||
# install containerd
|
||||
&& guestfish --remote command "apt-get update -q" \
|
||||
&& guestfish --remote command "apt-get install -yq containerd.io" \
|
||||
# configure containerd
|
||||
&& guestfish --remote command "mkdir -p /etc/containerd" \
|
||||
&& guestfish --remote sh "containerd config default | tee /etc/containerd/config.toml" \
|
||||
&& guestfish --remote command "sed -i '/SystemdCgroup/ s/=.*/= true/' /etc/containerd/config.toml" \
|
||||
&& guestfish --remote command "containerd config dump >/dev/null" \
|
||||
# install kubernetes
|
||||
&& guestfish --remote command "apt-get install -yq kubelet kubeadm" \
|
||||
# clean apt cache
|
||||
&& guestfish --remote sh 'apt-get clean && rm -rf /var/lib/apt/lists/*' \
|
||||
# write system configuration
|
||||
&& guestfish --remote sh 'printf "%s\n" net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.all.disable_ipv6=0 net.ipv4.tcp_congestion_control=bbr vm.overcommit_memory=1 kernel.panic=10 kernel.panic_on_oops=1 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=524288 | tee > /etc/sysctl.d/kubernetes.conf' \
|
||||
&& guestfish --remote sh 'printf "%s\n" overlay br_netfilter | tee /etc/modules-load.d/kubernetes.conf' \
|
||||
&& guestfish --remote sh "rm -f /etc/resolv.conf && ln -s ../run/systemd/resolve/stub-resolv.conf /etc/resolv.conf" \
|
||||
# umount all and exit
|
||||
&& guestfish --remote umount-all \
|
||||
&& guestfish --remote exit
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /build/image.img /disk/image.qcow2
|
||||
@@ -50,6 +50,21 @@ app.kubernetes.io/name: {{ include "kubernetes.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
DNS domain used INSIDE the tenant cluster (kubelet --cluster-domain,
|
||||
apiserver --service-cluster-ip-range FQDNs, CoreDNS authoritative zone).
|
||||
Pinned to Kamaji's default `networkProfile.clusterDomain` since this chart
|
||||
does not currently expose a knob to override it. If that ever becomes
|
||||
configurable, plumb the override here and every consumer picks it up.
|
||||
|
||||
Distinct from .Values._cluster["cluster-domain"], which is the MANAGEMENT
|
||||
cluster domain (e.g. cozy.local) where the Kamaji control plane and
|
||||
monitoring stack live.
|
||||
*/}}
|
||||
{{- define "kubernetes.tenantClusterDomain" -}}
|
||||
cluster.local
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
wait-for-kubeconfig init container shared by the control-plane-side
|
||||
Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the
|
||||
@@ -59,10 +74,10 @@ Kamaji is still bootstrapping; this container polls the mounted path and
|
||||
exits only when super-admin.svc appears, which happens after kubelet's
|
||||
optional-Secret refresh cycle.
|
||||
|
||||
The 10m deadline stays strictly below the 15m HelmRelease
|
||||
The 10m deadline stays strictly below the 20m HelmRelease
|
||||
Install.Timeout set by cozystack-api for the Kubernetes kind (via the
|
||||
release.cozystack.io/helm-install-timeout annotation) so the
|
||||
CrashLoopBackOff surfaces before flux remediation fires and uninstalls
|
||||
release.cozystack.io/helm-install-timeout annotation on the cozyrds
|
||||
entry) so the CrashLoopBackOff surfaces before flux remediation fires and uninstalls
|
||||
the Cluster CR.
|
||||
|
||||
The default image lives in images/busybox.tag and points directly at
|
||||
|
||||
@@ -1,3 +1,23 @@
|
||||
{{- /*
|
||||
Talos -> Kubernetes support matrix. Each Talos minor release supports a
|
||||
bounded window of Kubernetes minor versions; running a kubelet outside
|
||||
that window produces a silently broken Talos+kubelet combination that
|
||||
HelmRelease cannot detect. Fail the chart render instead, so the operator
|
||||
sees the mismatch in Flux/dashboards before workers roll over.
|
||||
|
||||
The matrix only fires for Talos versions it knows about; operators who
|
||||
override talos.version to something outside this list take responsibility
|
||||
for compatibility themselves (extend the dict or upstream a PR).
|
||||
Source: https://docs.siderolabs.com/talos/v1.13/getting-started/support-matrix
|
||||
*/}}
|
||||
{{- $talosK8sSupportMatrix := dict
|
||||
"v1.13.0" (list "v1.31" "v1.32" "v1.33" "v1.34" "v1.35" "v1.36")
|
||||
}}
|
||||
{{- with index $talosK8sSupportMatrix .Values.talos.version }}
|
||||
{{- if not (has $.Values.version .) }}
|
||||
{{- fail (printf "Kubernetes %s is not supported by Talos %s. Supported versions: %v. Bump tenant 'version' or set talos.version explicitly." $.Values.version $.Values.talos.version .) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $etcd := .Values._namespace.etcd }}
|
||||
{{- /*
|
||||
When $etcd is empty, the parent Tenant application has not populated
|
||||
@@ -10,13 +30,38 @@
|
||||
flux marks the HelmRelease Ready and retries its 5m reconcile loop until
|
||||
the Tenant chart catches up.
|
||||
*/}}
|
||||
{{- /* The TalosConfigTemplate and the apiserver-Service ClusterIP-SAN
|
||||
used to be rendered in this template behind a lookup gate that
|
||||
races every fresh install: Service / Talos CA / K8s CA do not
|
||||
exist on first Helm render, so the gate fails, the TCT is
|
||||
skipped, and Helm's release-storage three-way merge then refuses
|
||||
to upgrade because no values changed. Both objects are now
|
||||
produced by the post-install/post-upgrade Helm hook Job in
|
||||
templates/talos/talos-reconcile-job.yaml, which blocks until the
|
||||
four dependencies exist and then `kubectl apply`s deterministic
|
||||
results. The Job owns the TCT through an ownerReference to the
|
||||
KamajiControlPlane so K8s garbage-collects it naturally on
|
||||
release uninstall. */}}
|
||||
{{- /*
|
||||
Cluster pod/service CIDRs MUST match between the CAPI Cluster
|
||||
spec.clusterNetwork below and the Talos worker machineconfig's
|
||||
cluster.network — Talos surfaces overlaps against the host pod CIDR
|
||||
as a diagnostic at boot, and divergence between CAPI and Talos would
|
||||
manifest as silently broken kube-proxy / Cilium routing. Keep them
|
||||
in one place here.
|
||||
*/}}
|
||||
{{- $podCIDR := "10.243.0.0/16" }}
|
||||
{{- $serviceCIDR := "10.95.0.0/16" }}
|
||||
{{- $ingress := .Values._namespace.ingress }}
|
||||
{{- $host := .Values._namespace.host }}
|
||||
{{- $kubevirtmachinetemplateNames := list }}
|
||||
{{- define "kubevirtmachinetemplate" -}}
|
||||
spec:
|
||||
{{- /* Talos has no SSH; use 'none' so CAPK does not block Machine.Ready
|
||||
on an SSH probe that will never succeed. The Machine reaches Ready
|
||||
once kubelet registers with the tenant apiserver via standard CSR. */}}
|
||||
virtualMachineBootstrapCheck:
|
||||
checkStrategy: ssh
|
||||
checkStrategy: none
|
||||
virtualMachineTemplate:
|
||||
metadata:
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
@@ -33,16 +78,21 @@ spec:
|
||||
runStrategy: Always
|
||||
dataVolumeTemplates:
|
||||
- metadata:
|
||||
name: disk-kubelet
|
||||
name: disk-system
|
||||
annotations:
|
||||
{{/* Populator creates intermediate PVC + PV rebind that causes ClaimMisbound with LINSTOR CSI */}}
|
||||
cdi.kubevirt.io/storage.usePopulator: "false"
|
||||
spec:
|
||||
{{- /* Use the openstack/configdrive Talos image: KubeVirt CAPK
|
||||
injects bootstrap data via cloudInitConfigDrive (OpenStack
|
||||
metadata service shape), and the openstack platform image
|
||||
is the one that reads it. The nocloud-amd64 image silently
|
||||
ignores configdrive and never applies the machineconfig. */}}
|
||||
source:
|
||||
http:
|
||||
url: https://factory.talos.dev/image/{{ $.Values.talos.schematicID }}/{{ $.Values.talos.version }}/openstack-amd64.raw.xz
|
||||
{{- if hasKey .group "ephemeralStorage" }}
|
||||
{{- fail (printf "nodeGroup %q: ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system." .groupName) }}
|
||||
{{- end }}
|
||||
source:
|
||||
blank: {}
|
||||
storage:
|
||||
resources:
|
||||
requests:
|
||||
@@ -92,13 +142,10 @@ spec:
|
||||
disk:
|
||||
bus: virtio
|
||||
pciAddress: 0000:07:00.0
|
||||
- name: disk-kubelet
|
||||
disk:
|
||||
bus: virtio
|
||||
pciAddress: 0000:08:00.0
|
||||
blockSize:
|
||||
matchVolume:
|
||||
enabled: true
|
||||
custom:
|
||||
logical: 512
|
||||
physical: 4096
|
||||
interfaces:
|
||||
- name: default
|
||||
bridge: {}
|
||||
@@ -112,11 +159,8 @@ spec:
|
||||
evictionStrategy: External
|
||||
volumes:
|
||||
- name: system
|
||||
containerDisk:
|
||||
image: "{{ $.Files.Get (printf "images/ubuntu-container-disk-%s.tag" $.Values.version) | trim }}"
|
||||
- name: disk-kubelet
|
||||
dataVolume:
|
||||
name: disk-kubelet
|
||||
name: disk-system
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
@@ -151,10 +195,10 @@ spec:
|
||||
clusterNetwork:
|
||||
pods:
|
||||
cidrBlocks:
|
||||
- 10.243.0.0/16
|
||||
- {{ $podCIDR }}
|
||||
services:
|
||||
cidrBlocks:
|
||||
- 10.95.0.0/16
|
||||
- {{ $serviceCIDR }}
|
||||
controlPlaneRef:
|
||||
namespace: {{ .Release.Namespace }}
|
||||
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
|
||||
@@ -208,10 +252,69 @@ spec:
|
||||
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
|
||||
hostname: {{ .Values.host | default (printf "%s.%s" .Release.Name $host) }}:443
|
||||
className: "{{ $ingress }}"
|
||||
additionalServicePorts:
|
||||
- name: talos-trustd
|
||||
port: 50001
|
||||
targetPort: 50001
|
||||
protocol: TCP
|
||||
{{- /* Initial render lists only the DNS SANs because the apiserver
|
||||
Service ClusterIP is not yet allocated when Helm executes the
|
||||
template. The talos-reconcile post-install/post-upgrade hook
|
||||
re-patches certSANs with the live ClusterIP once Kamaji emits
|
||||
the Service, and Kamaji then re-issues the apiserver TLS cert
|
||||
to cover it. */}}
|
||||
certSANs:
|
||||
- {{ printf "%s.%s.svc" .Release.Name .Release.Namespace }}
|
||||
- {{ printf "%s.%s.svc.%s" .Release.Name .Release.Namespace (include "kubernetes.tenantClusterDomain" .) }}
|
||||
deployment:
|
||||
podAdditionalMetadata:
|
||||
labels:
|
||||
policy.cozystack.io/allow-to-etcd: "true"
|
||||
extraContainers:
|
||||
- name: talos-csr-signer
|
||||
image: "{{ default (.Files.Get "images/talos-csr-signer.tag" | trim) .Values.images.talosCsrSigner }}"
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: trustd
|
||||
containerPort: 50001
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: TALOS_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Release.Name }}-talos-secrets
|
||||
key: token
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- name: talos-ca
|
||||
mountPath: /etc/talos-ca
|
||||
readOnly: true
|
||||
- name: talos-tls-cert
|
||||
mountPath: /etc/talos-server-crt
|
||||
readOnly: true
|
||||
extraVolumes:
|
||||
- name: talos-ca
|
||||
secret:
|
||||
secretName: {{ .Release.Name }}-talos-ca
|
||||
- name: talos-tls-cert
|
||||
secret:
|
||||
secretName: {{ .Release.Name }}-talos-tls-cert
|
||||
replicas: {{ .Values.controlPlane.replicas }}
|
||||
version: {{ include "kubernetes.versionMap" $ }}
|
||||
---
|
||||
@@ -406,104 +509,18 @@ metadata:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmConfigTemplate
|
||||
metadata:
|
||||
name: {{ $.Release.Name }}-{{ $groupName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
files:
|
||||
- path: /usr/bin/update-k8s.sh
|
||||
owner: root:root
|
||||
permissions: "0755"
|
||||
content: |
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Expected to be passed in via preKubeadmCommands
|
||||
: "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}"
|
||||
|
||||
ARCH="$(uname -m)"
|
||||
case "${ARCH}" in
|
||||
x86_64) ARCH=amd64 ;;
|
||||
aarch64) ARCH=arm64 ;;
|
||||
esac
|
||||
|
||||
# Use your internal mirror here for real-world use.
|
||||
BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}"
|
||||
|
||||
echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..."
|
||||
curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet
|
||||
curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm
|
||||
chmod 0755 /root/kubelet
|
||||
chmod 0755 /root/kubeadm
|
||||
if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi
|
||||
if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi
|
||||
{{- $sec := lookup "v1" "Secret" $.Release.Namespace (printf "%s-patch-containerd" $.Release.Name) }}
|
||||
{{- if $sec }}
|
||||
{{- range $key, $_ := $sec.data }}
|
||||
- path: /etc/containerd/certs.d/{{ trimSuffix ".toml" $key }}/hosts.toml
|
||||
contentFrom:
|
||||
secret:
|
||||
name: {{ $.Release.Name }}-patch-containerd
|
||||
key: {{ $key }}
|
||||
permissions: "0400"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
diskSetup:
|
||||
filesystems:
|
||||
- device: /dev/vdb
|
||||
filesystem: xfs
|
||||
label: persistent
|
||||
partition: "none"
|
||||
{{/* overwrite must stay false: XFS on /dev/vdb is created on first boot; subsequent reboots of the same VM must reuse the existing filesystem via LABEL=persistent */}}
|
||||
overwrite: false
|
||||
mounts:
|
||||
- ["LABEL=persistent", "/persistent"]
|
||||
- ["/persistent/kubelet", "/var/lib/kubelet", "none", "bind,nofail"]
|
||||
- ["/persistent/containerd", "/var/lib/containerd", "none", "bind,nofail"]
|
||||
preKubeadmCommands:
|
||||
- KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true
|
||||
- sed -i 's|root:x:|root::|' /etc/passwd
|
||||
- systemctl stop containerd.service
|
||||
- mkdir -p /persistent/kubelet /persistent/containerd /var/lib/kubelet /var/lib/containerd
|
||||
- mountpoint -q /var/lib/kubelet || mount -o bind /persistent/kubelet /var/lib/kubelet
|
||||
- mountpoint -q /var/lib/containerd || mount -o bind /persistent/containerd /var/lib/containerd
|
||||
{{/* containerd 2.x renamed the CRI plugin from io.containerd.grpc.v1.cri to io.containerd.cri.v1.images; both seds run unconditionally and the wrong-version one is a no-op */}}
|
||||
- sed -i '/\[plugins\.[^]]*grpc\.v1\.cri[^]]*registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml
|
||||
- sed -i '/\[plugins\.[^]]*io\.containerd\.cri\.v1\.images[^]]*registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml
|
||||
- systemctl start containerd.service
|
||||
joinConfiguration:
|
||||
nodeRegistration:
|
||||
kubeletExtraArgs:
|
||||
{{- /*
|
||||
GPU node-groups: label every node `gpu=on` so HAMi's
|
||||
hami-device-plugin DaemonSet (nodeSelector gpu=on) schedules
|
||||
and advertises nvidia.com/gpu. Without the label the plugin
|
||||
stays at DESIRED=0 and no GPUs are exposed to the tenant.
|
||||
*/}}
|
||||
{{- if $group.gpus }}
|
||||
node-labels: "gpu=on"
|
||||
{{- end }}
|
||||
system-reserved: "memory={{ $systemReservedMemory }},cpu={{ $systemReservedCpu }}"
|
||||
kube-reserved: "memory={{ $kubeReservedMemory }},cpu={{ $kubeReservedCpu }}"
|
||||
eviction-hard: "memory.available<{{ $evictionHardMemory }},nodefs.available<10%,imagefs.available<15%,nodefs.inodesFree<5%"
|
||||
eviction-soft: "memory.available<{{ $evictionSoftMemory }},nodefs.available<15%,imagefs.available<20%"
|
||||
eviction-soft-grace-period: "memory.available=1m30s,nodefs.available=1m30s,imagefs.available=1m30s"
|
||||
# 256Mi ensures meaningful reclamation per eviction cycle while being
|
||||
# proportional to the 256Mi-1Gi auto-computed reservation range.
|
||||
eviction-minimum-reclaim: "memory.available=256Mi"
|
||||
enforce-node-allocatable: "pods"
|
||||
ignorePreflightErrors:
|
||||
- FileExisting-conntrack
|
||||
discovery:
|
||||
bootstrapToken:
|
||||
apiServerEndpoint: {{ $.Release.Name }}.{{ $.Release.Namespace }}.svc:6443
|
||||
initConfiguration:
|
||||
skipPhases:
|
||||
- addon/kube-proxy
|
||||
{{- /* TalosConfigTemplate is no longer rendered here — the chart cannot
|
||||
produce a valid Talos worker machineconfig at Helm template time
|
||||
because three of the inputs (apiserver Service ClusterIP, Talos
|
||||
CA from cert-manager, Kamaji-issued tenant CA) are created
|
||||
asynchronously after Helm applies this manifest. The
|
||||
talos-reconcile-job hook (templates/talos/talos-reconcile-job.yaml)
|
||||
waits for those inputs and produces the TalosConfigTemplate with
|
||||
a KamajiControlPlane ownerReference for natural GC. The
|
||||
MachineDeployment below still references the TCT name; CAPI's
|
||||
Machine controller blocks on its absence and unblocks the moment
|
||||
the hook applies it, which happens in the same post-install/
|
||||
post-upgrade cycle. */}}
|
||||
---
|
||||
{{- $context := deepCopy $ }}
|
||||
{{- $_ := set $context "group" $group }}
|
||||
@@ -569,8 +586,8 @@ spec:
|
||||
spec:
|
||||
bootstrap:
|
||||
configRef:
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
|
||||
kind: KubeadmConfigTemplate
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
|
||||
kind: TalosConfigTemplate
|
||||
name: {{ $.Release.Name }}-{{ $groupName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
clusterName: {{ $.Release.Name }}
|
||||
@@ -588,8 +605,13 @@ metadata:
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
spec:
|
||||
clusterName: {{ $.Release.Name }}
|
||||
maxUnhealthy: 0
|
||||
nodeStartupTimeout: 10m
|
||||
{{- /* MHC spec.maxUnhealthy is an IntOrString. CAPI admission accepts a
|
||||
bare integer (0, 1, ...) or a percentage string ("0%", "50%"), but
|
||||
rejects a quoted plain integer ("0"). Quote only when the operator
|
||||
supplied a percentage; otherwise render the value as a bare int so
|
||||
the admission webhook is happy with the chart-default "0". */}}
|
||||
maxUnhealthy: {{ if hasSuffix "%" (toString $.Values.nodeHealthCheck.maxUnhealthy) }}{{ $.Values.nodeHealthCheck.maxUnhealthy | quote }}{{ else }}{{ $.Values.nodeHealthCheck.maxUnhealthy | int }}{{ end }}
|
||||
nodeStartupTimeout: {{ $.Values.nodeHealthCheck.nodeStartupTimeout }}
|
||||
selector:
|
||||
matchLabels:
|
||||
cluster.x-k8s.io/cluster-name: {{ $.Release.Name }}
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
{{- $sourceSecret := lookup "v1" "Secret" "cozy-system" "patch-containerd" }}
|
||||
{{- if $sourceSecret }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-patch-containerd
|
||||
namespace: {{ .Release.Namespace }}
|
||||
type: {{ $sourceSecret.type }}
|
||||
data:
|
||||
{{- range $key, $value := $sourceSecret.data }}
|
||||
{{ printf "%s: %s" $key ($value | quote) | indent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,6 +1,14 @@
|
||||
{{- define "cozystack.defaultCiliumValues" -}}
|
||||
{{- /* Cilium runs hostNetwork on the Talos worker nodes and resolves the
|
||||
apiserver via the worker's /etc/hosts, which the talos-reconcile
|
||||
post-install hook seeds with the Kamaji Service ClusterIP under
|
||||
<release>.<ns>.svc through machine.network.extraHostEntries. So a
|
||||
plain DNS host works here without a lookup-and-reuse race: the
|
||||
cert covers the same DNS name from chart render time, and the
|
||||
hosts entry is set well before kubelet (and therefore Cilium)
|
||||
comes up on the node. */}}
|
||||
cilium:
|
||||
k8sServiceHost: {{ .Release.Name }}.{{ .Release.Namespace }}.svc
|
||||
k8sServiceHost: {{ printf "%s.%s.svc" .Release.Name .Release.Namespace }}
|
||||
k8sServicePort: 6443
|
||||
routingMode: tunnel
|
||||
MTU: 1350
|
||||
|
||||
@@ -41,7 +41,7 @@ ouroboros:
|
||||
federations, custom kubelet --cluster-domain, etc.) can
|
||||
override via
|
||||
`addons.ouroboros.valuesOverride.ouroboros.controller.clusterDomain`. */}}
|
||||
clusterDomain: cluster.local
|
||||
clusterDomain: {{ include "kubernetes.tenantClusterDomain" $ }}
|
||||
{{- end }}
|
||||
|
||||
{{- /* addons.ouroboros.enabled requires addons.ingressNginx.enabled. The
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
{{- /*
|
||||
Job that materialises a long-lived kubeadm-format bootstrap-token Secret
|
||||
inside the tenant kube-system namespace, so that Talos workers can use
|
||||
cluster.token from their machineconfig to perform kubelet CSR.
|
||||
|
||||
Why a Job and not a kube-native Secret on the management cluster:
|
||||
bootstrap-token-<id> Secrets must live in the tenant apiserver's
|
||||
kube-system, which the management cluster can only reach via the
|
||||
Kamaji-issued admin kubeconfig stored as <release>-admin-kubeconfig.
|
||||
|
||||
The Job runs as a Helm post-install/post-upgrade hook so it executes
|
||||
after the KamajiControlPlane has reconciled and the admin kubeconfig
|
||||
Secret exists. It is idempotent (kubectl apply) and re-runs harmlessly
|
||||
on every upgrade.
|
||||
*/}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-bootstrap-token-tenant
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "0"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-bootstrap-token-tenant
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "10"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
# backoffLimit + restartPolicy: OnFailure gives the Job up to 11 attempts at
|
||||
# the tenant apiserver with exponential backoff (capped at 6 min between
|
||||
# restarts). Worst case ~15 min, well inside the chart's HR install timeout
|
||||
# (release.cozystack.io/helm-install-timeout: 20m on the cozyrds entry) so
|
||||
# the Job either succeeds within the HR window or fails the chart upgrade
|
||||
# loudly, instead of letting Helm mark the install "successful" while the
|
||||
# bootstrap-token Secret never lands in the tenant kube-system.
|
||||
backoffLimit: 10
|
||||
ttlSecondsAfterFinished: 600
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ .Release.Name }}-bootstrap-token-tenant
|
||||
containers:
|
||||
- name: kubectl
|
||||
image: "{{ default (.Files.Get "images/kubectl.tag" | trim) .Values.images.kubectl }}"
|
||||
env:
|
||||
- name: KUBECONFIG
|
||||
value: /tenant-admin/super-admin.svc
|
||||
- name: TOKEN_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Release.Name }}-talos-secrets
|
||||
key: bootstrapTokenId
|
||||
- name: TOKEN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Release.Name }}-talos-secrets
|
||||
key: bootstrapTokenSecret
|
||||
volumeMounts:
|
||||
- name: tenant-admin
|
||||
mountPath: /tenant-admin
|
||||
readOnly: true
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
# The kubectl apply calls below fail with a non-zero exit when
|
||||
# the tenant apiserver is unreachable, which restarts the pod
|
||||
# under backoffLimit. No internal retry loop needed — Job-level
|
||||
# backoff handles the transient unreadiness window after Kamaji
|
||||
# bootstraps the konnectivity plane.
|
||||
kubectl version --request-timeout=10s
|
||||
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: bootstrap-token-${TOKEN_ID}
|
||||
namespace: kube-system
|
||||
type: bootstrap.kubernetes.io/token
|
||||
stringData:
|
||||
token-id: ${TOKEN_ID}
|
||||
token-secret: ${TOKEN_SECRET}
|
||||
description: "Talos worker bootstrap token, managed by cozystack kubernetes chart"
|
||||
usage-bootstrap-authentication: "true"
|
||||
usage-bootstrap-signing: "true"
|
||||
auth-extra-groups: "system:bootstrappers:kubeadm:default-node-token"
|
||||
EOF
|
||||
|
||||
echo "[$(date -u +%H:%M:%S)] bootstrap-token-${TOKEN_ID} ensured"
|
||||
|
||||
# Grant system:nodes the ability to watch the kubernetes
|
||||
# Endpoint/EndpointSlice in the default namespace. Talos
|
||||
# workers run an internal k8s.EndpointController (apiserver
|
||||
# failover discovery) that otherwise spams
|
||||
# "endpointslices forbidden" warnings.
|
||||
# Note: resourceNames intentionally omitted — Talos's
|
||||
# informer issues unbounded list/watch and Kubernetes RBAC
|
||||
# only honours resourceNames on list/watch when the client
|
||||
# restricts via a field selector, which Talos does not.
|
||||
# The Role is namespace-scoped to default, where only the
|
||||
# built-in "kubernetes" endpoint lives.
|
||||
cat <<'EOF' | kubectl apply -f -
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: talos:node-apiserver-endpoint-discovery
|
||||
namespace: default
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: talos:node-apiserver-endpoint-discovery
|
||||
namespace: default
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: talos:node-apiserver-endpoint-discovery
|
||||
subjects:
|
||||
- kind: Group
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
name: system:nodes
|
||||
EOF
|
||||
|
||||
# Drop the previous ClusterRole/Binding incarnation if it
|
||||
# exists from earlier deploys of this chart.
|
||||
kubectl delete clusterrole talos:node-apiserver-endpoint-discovery --ignore-not-found
|
||||
kubectl delete clusterrolebinding talos:node-apiserver-endpoint-discovery --ignore-not-found
|
||||
|
||||
echo "[$(date -u +%H:%M:%S)] talos:node-apiserver-endpoint-discovery RBAC ensured"
|
||||
volumes:
|
||||
- name: tenant-admin
|
||||
secret:
|
||||
secretName: {{ .Release.Name }}-admin-kubeconfig
|
||||
@@ -0,0 +1,495 @@
|
||||
{{- /*
|
||||
Post-install/post-upgrade Helm hook that resolves the chart's
|
||||
lookup-and-reuse race for TalosConfigTemplate and
|
||||
KamajiControlPlane.spec.network.certSANs.
|
||||
|
||||
Background. The Talos worker machineconfig and the apiserver Service
|
||||
ClusterIP-SAN both need values that only exist AFTER Helm has applied
|
||||
the first render: the Service is emitted by the Kamaji controller in
|
||||
reaction to the KamajiControlPlane CR, the Talos-CA TLS Secret is
|
||||
issued by cert-manager from the Talos PKI Certificate CR, and the
|
||||
Kubernetes-CA Secret is generated by Kamaji as part of its tenant
|
||||
control-plane bootstrap. A Helm template that uses `lookup` to read
|
||||
any of these into the rendered manifest fails the race on every fresh
|
||||
install — the lookup returns nil, the gated TalosConfigTemplate is
|
||||
skipped, and Helm's release-storage three-way merge concludes the
|
||||
release is in-sync on every subsequent reconcile because no chart
|
||||
value changed. The result is a tenant whose KamajiControlPlane is
|
||||
Ready but whose CAPI MachineDeployment is stuck on
|
||||
`cannot create a new MachineSet when templates do not exist` forever.
|
||||
|
||||
This Job runs once Helm has applied everything else in the release,
|
||||
blocks until the four dependencies are observably present, then
|
||||
produces both objects via `kubectl apply` (idempotent). The
|
||||
TalosConfigTemplate carries an ownerReference to the KamajiControlPlane
|
||||
so Kubernetes garbage-collects it through the standard owner-reference
|
||||
cascade when the chart's release is uninstalled — Helm itself does not
|
||||
need to track the resource, and a `pre-delete` hook is not required.
|
||||
The certSANs patch is a merge against the KCP `spec.network.certSANs`
|
||||
field; the Kamaji controller picks up the change and re-issues the
|
||||
apiserver TLS cert with the live ClusterIP in its SAN list.
|
||||
|
||||
Hook weight 5 places the Job after Helm has emitted the
|
||||
KamajiControlPlane / Talos PKI Certificate / talos-secrets but before
|
||||
the bootstrap-token-tenant Job (weight 10), so workers see a
|
||||
reachable apiserver, a SAN-correct TLS cert, and a populated
|
||||
bootstrap-token Secret in the tenant kube-system the first time
|
||||
they boot.
|
||||
*/}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
resourceNames: ["{{ .Release.Name }}"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames:
|
||||
- "{{ .Release.Name }}-talos-secrets"
|
||||
- "{{ .Release.Name }}-talos-ca"
|
||||
- "{{ .Release.Name }}-ca"
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["controlplane.cluster.x-k8s.io"]
|
||||
resources: ["kamajicontrolplanes"]
|
||||
resourceNames: ["{{ .Release.Name }}"]
|
||||
verbs: ["get", "patch"]
|
||||
- apiGroups: ["bootstrap.cluster.x-k8s.io"]
|
||||
resources: ["talosconfigtemplates"]
|
||||
verbs: ["get", "create", "patch", "update"]
|
||||
---
|
||||
{{- /* The tenant-root chart installs a default-deny egress policy on every
|
||||
pod in the tenant namespace; without an explicit allow, this hook Job
|
||||
cannot reach the management-cluster apiserver to read the Service
|
||||
ClusterIP / Secrets / KamajiControlPlane it has to reconcile. Cilium
|
||||
policies are additive, so this rule layers on top of the tenant
|
||||
default-deny without weakening it for anything else. Endpoint
|
||||
selector is scoped to pods carrying the hook label only. */}}
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
cozystack.io/talos-reconcile: "{{ .Release.Name }}"
|
||||
egress:
|
||||
- toEntities:
|
||||
- kube-apiserver
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": kube-system
|
||||
"k8s:k8s-app": kube-dns
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "53"
|
||||
protocol: UDP
|
||||
- port: "53"
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
{{- /* Cluster-scoped read on the management CoreDNS Service so the hook
|
||||
can look up its ClusterIP at runtime and inject it into the worker
|
||||
machineconfig as a host-level nameserver. Tenant workers run as
|
||||
KubeVirt VMs inside the management pod network; tenant in-cluster
|
||||
pods still use kubelet --cluster-dns (tenant CoreDNS) via
|
||||
dnsPolicy: ClusterFirst, but the worker host itself (kubelet's
|
||||
mount syscall path, image pulls) consults /etc/resolv.conf which
|
||||
Talos populates from machine.network.nameservers. Without
|
||||
management CoreDNS as host resolver, mounting NFS from
|
||||
kubevirt-csi (linstor-csi-nfs.cozy-linstor.svc) fails with
|
||||
NXDOMAIN.
|
||||
|
||||
Ubuntu+kubeadm workers in main inherit pod DNS via cloud-init
|
||||
and got management CoreDNS for free; Talos reads only what is
|
||||
in machineconfig, so the chart has to plumb it explicitly.
|
||||
|
||||
resourceNames pins the grant to the single well-known Service
|
||||
name kube-dns — cluster-wide get on services scoped to that
|
||||
single name is functionally a single-resource permission. */}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
resourceNames: ["kube-dns"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns
|
||||
labels:
|
||||
{{- include "kubernetes.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "1"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ .Release.Namespace }}-{{ .Release.Name }}-talos-reconcile-coredns
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-talos-reconcile
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
{{- $kubeletVersion := include "kubernetes.versionMap" $ | trim }}
|
||||
{{- $talosVersion := $.Values.talos.version }}
|
||||
{{- $podCIDR := "10.243.0.0/16" }}
|
||||
{{- $serviceCIDR := "10.95.0.0/16" }}
|
||||
{{- $dnsDomain := include "kubernetes.tenantClusterDomain" $ }}
|
||||
{{- /* Management cluster DNS domain. Tenant Kubernetes workers run as
|
||||
KubeVirt VMs inside the management cluster's pod network and need
|
||||
to resolve management-cluster service short names ("<svc>.<ns>.svc")
|
||||
the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The
|
||||
kubevirt-csi NFS mount path is the load-bearing case: kubelet on
|
||||
the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc`
|
||||
(no domain suffix), and without searchDomains in /etc/resolv.conf
|
||||
the host resolver hands that name to management CoreDNS verbatim
|
||||
and gets NXDOMAIN.
|
||||
|
||||
Distinct from tenantClusterDomain ("cluster.local"), which is the
|
||||
*tenant* kubelet --cluster-dns zone. This value is the
|
||||
*management* cluster-domain, sourced from .Values._cluster
|
||||
(populated by cozystack-controller from the platform config). */}}
|
||||
{{- $mgmtClusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }}
|
||||
{{- range $groupName, $group := .Values.nodeGroups }}
|
||||
{{- /* Mirror the kubelet-reservation computation from cluster.yaml so the
|
||||
worker machineconfig the hook applies carries the same numbers the
|
||||
MachineDeployment validator was happy with. Auto-computed
|
||||
system/kube reserved memory = 5% of effective memory clamped to
|
||||
[256Mi, 1Gi]; auto-computed system/kube reserved CPU = 5% of
|
||||
effective CPU clamped to [50m, 500m]. instanceType lookup is
|
||||
optional — operators that override resources.cpu / resources.memory
|
||||
on the nodeGroup get those, otherwise the chart falls back to the
|
||||
same static-defaults pair the cluster.yaml validator uses on a
|
||||
no-instanceType setup. */}}
|
||||
{{- $instanceType := dict }}
|
||||
{{- if $group.instanceType }}
|
||||
{{- $instanceType = lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType }}
|
||||
{{- $instanceType = $instanceType | default dict }}
|
||||
{{- end }}
|
||||
{{- $effectiveMemory := "" }}
|
||||
{{- if and $group.resources $group.resources.memory }}
|
||||
{{- $effectiveMemory = $group.resources.memory | toString }}
|
||||
{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }}
|
||||
{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }}
|
||||
{{- end }}
|
||||
{{- $effectiveCpu := "" }}
|
||||
{{- if and $group.resources $group.resources.cpu }}
|
||||
{{- $effectiveCpu = $group.resources.cpu | toString }}
|
||||
{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }}
|
||||
{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }}
|
||||
{{- end }}
|
||||
{{- $autoReservedMi := 256 }}
|
||||
{{- if $effectiveMemory }}
|
||||
{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }}
|
||||
{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }}
|
||||
{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }}
|
||||
{{- end }}
|
||||
{{- $autoReservedMillicores := 50 }}
|
||||
{{- if $effectiveCpu }}
|
||||
{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }}
|
||||
{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }}
|
||||
{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }}
|
||||
{{- end }}
|
||||
{{- $kubeletOverride := $group.kubelet | default dict }}
|
||||
{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }}
|
||||
{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }}
|
||||
{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }}
|
||||
{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }}
|
||||
{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }}
|
||||
{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ $.Release.Name }}-talos-reconcile-{{ $groupName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "kubernetes.labels" $ | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "5"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
backoffLimit: 5
|
||||
ttlSecondsAfterFinished: 600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
cozystack.io/talos-reconcile: {{ $.Release.Name | quote }}
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: {{ $.Release.Name }}-talos-reconcile
|
||||
containers:
|
||||
- name: kubectl
|
||||
image: "{{ default ($.Files.Get "images/kubectl.tag" | trim) $.Values.images.kubectl }}"
|
||||
env:
|
||||
- name: RELEASE
|
||||
value: {{ $.Release.Name | quote }}
|
||||
- name: NS
|
||||
value: {{ $.Release.Namespace | quote }}
|
||||
- name: GROUP_NAME
|
||||
value: {{ $groupName | quote }}
|
||||
- name: TALOS_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Release.Name }}-talos-secrets
|
||||
key: token
|
||||
- name: CLUSTER_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Release.Name }}-talos-secrets
|
||||
key: clusterId
|
||||
- name: CLUSTER_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Release.Name }}-talos-secrets
|
||||
key: clusterSecret
|
||||
- name: BOOTSTRAP_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $.Release.Name }}-talos-secrets
|
||||
key: bootstrapToken
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
echo "[$(date -u +%H:%M:%S)] talos-reconcile/${GROUP_NAME}: waiting for Service ClusterIP, Talos CA, K8s CA, management CoreDNS"
|
||||
for i in $(seq 1 120); do
|
||||
SVC_IP=$(kubectl -n "$NS" get svc "$RELEASE" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)
|
||||
TALOS_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-talos-ca" -o jsonpath='{.data.tls\.crt}' 2>/dev/null || true)
|
||||
K8S_CA_B64=$(kubectl -n "$NS" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true)
|
||||
COREDNS_IP=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)
|
||||
if [ -n "$SVC_IP" ] && [ -n "$TALOS_CA_B64" ] && [ -n "$K8S_CA_B64" ] && [ -n "$COREDNS_IP" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if [ -z "$SVC_IP" ] || [ -z "$TALOS_CA_B64" ] || [ -z "$K8S_CA_B64" ] || [ -z "$COREDNS_IP" ]; then
|
||||
echo "[$(date -u +%H:%M:%S)] timed out waiting for lookup inputs (svc='$SVC_IP' talos-ca=$([ -n "$TALOS_CA_B64" ] && echo ok) k8s-ca=$([ -n "$K8S_CA_B64" ] && echo ok) coredns='$COREDNS_IP')"
|
||||
exit 1
|
||||
fi
|
||||
echo "[$(date -u +%H:%M:%S)] inputs ready (svc=${SVC_IP} coredns=${COREDNS_IP})"
|
||||
|
||||
# Patch KamajiControlPlane.spec.network.certSANs so the apiserver
|
||||
# TLS cert covers the live Service ClusterIP that Talos workers
|
||||
# dial. JSON-merge-patch is array-replace semantics, which is what
|
||||
# we want here — Kamaji owns the full SAN list.
|
||||
kubectl -n "$NS" patch kamajicontrolplane "$RELEASE" --type=merge --patch "$(
|
||||
cat <<JSON
|
||||
{"spec":{"network":{"certSANs":["${RELEASE}.${NS}.svc","${RELEASE}.${NS}.svc.{{ $dnsDomain }}","${SVC_IP}"]}}}
|
||||
JSON
|
||||
)"
|
||||
|
||||
# Talos workers dial the apiserver Service via the DNS name
|
||||
# (${RELEASE}.${NS}.svc) instead of the live ClusterIP, with
|
||||
# machine.network.extraHostEntries injecting the IP into the
|
||||
# worker's /etc/hosts. This avoids a TLS race against the
|
||||
# csr-signer sidecar: the Talos PKI Certificate is issued
|
||||
# with DNS SANs at chart render time and never needs to be
|
||||
# patched after the Service ClusterIP becomes known. The
|
||||
# apiserver Kubernetes TLS cert (handled by KCP certSANs above)
|
||||
# auto-reloads on Kamaji reissue, but the csr-signer container
|
||||
# loads its TLS cert once at startup and does not watch the
|
||||
# file — so any in-place cert rotation leaves it serving the
|
||||
# original (no-IP-SAN) cert until the Kamaji pod restarts.
|
||||
# Routing through DNS sidesteps the whole problem.
|
||||
|
||||
KCP_UID=$(kubectl -n "$NS" get kamajicontrolplane "$RELEASE" -o jsonpath='{.metadata.uid}')
|
||||
|
||||
# Apply the TalosConfigTemplate. ownerReference back to the KCP
|
||||
# makes the standard Kubernetes garbage collector clean it up on
|
||||
# release uninstall; Helm itself does not track this resource.
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
|
||||
kind: TalosConfigTemplate
|
||||
metadata:
|
||||
name: ${RELEASE}-${GROUP_NAME}
|
||||
namespace: ${NS}
|
||||
ownerReferences:
|
||||
- apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
|
||||
kind: KamajiControlPlane
|
||||
name: ${RELEASE}
|
||||
uid: ${KCP_UID}
|
||||
controller: true
|
||||
blockOwnerDeletion: true
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
generateType: none
|
||||
talosVersion: "{{ $talosVersion }}"
|
||||
data: |
|
||||
version: v1alpha1
|
||||
persist: true
|
||||
machine:
|
||||
type: worker
|
||||
token: ${TALOS_TOKEN}
|
||||
network:
|
||||
# Management CoreDNS as the worker's host
|
||||
# resolver, plus management cluster search
|
||||
# domains so partial names like
|
||||
# "linstor-csi-nfs.cozy-linstor.svc" resolve
|
||||
# the same way they do on Ubuntu+kubeadm
|
||||
# workers in main (which got these via
|
||||
# cloud-init/DHCP). Without searchDomains the
|
||||
# kubevirt-csi NFS mount fails NXDOMAIN
|
||||
# because mount.nfs hands the partial name
|
||||
# to the resolver verbatim and management
|
||||
# CoreDNS only knows full FQDNs.
|
||||
#
|
||||
# Tenant in-cluster pods are not affected:
|
||||
# dnsPolicy: ClusterFirst routes them through
|
||||
# kubelet --cluster-dns (tenant CoreDNS).
|
||||
# This block only configures host-side
|
||||
# resolution (kubelet mount syscalls, image
|
||||
# pulls, pods opted out of cluster DNS).
|
||||
nameservers:
|
||||
- ${COREDNS_IP}
|
||||
searchDomains:
|
||||
- svc.{{ $mgmtClusterDomain }}
|
||||
- {{ $mgmtClusterDomain }}
|
||||
{{- if ne $mgmtClusterDomain "cluster.local" }}
|
||||
- svc.cluster.local
|
||||
- cluster.local
|
||||
{{- end }}
|
||||
extraHostEntries:
|
||||
- ip: ${SVC_IP}
|
||||
aliases:
|
||||
- ${RELEASE}.${NS}.svc
|
||||
- ${RELEASE}.${NS}.svc.{{ $dnsDomain }}
|
||||
ca:
|
||||
crt: ${TALOS_CA_B64}
|
||||
{{- if $group.gpus }}
|
||||
# GPU node-groups: label every node `gpu=on` so
|
||||
# HAMi's hami-device-plugin DaemonSet (nodeSelector
|
||||
# gpu=on) schedules and advertises nvidia.com/gpu.
|
||||
# Without the label the plugin stays at DESIRED=0
|
||||
# and no GPUs are exposed to the tenant. Mirrors
|
||||
# main's kubeadm config (kubeletExtraArgs node-
|
||||
# labels: "gpu=on") so HAMi behaviour is identical
|
||||
# before and after the Talos worker rollover.
|
||||
nodeLabels:
|
||||
gpu: "on"
|
||||
{{- end }}
|
||||
kubelet:
|
||||
image: ghcr.io/siderolabs/kubelet:{{ $kubeletVersion }}
|
||||
extraConfig:
|
||||
systemReserved:
|
||||
cpu: "{{ $systemReservedCpu }}"
|
||||
memory: "{{ $systemReservedMemory }}"
|
||||
kubeReserved:
|
||||
cpu: "{{ $kubeReservedCpu }}"
|
||||
memory: "{{ $kubeReservedMemory }}"
|
||||
evictionHard:
|
||||
memory.available: "{{ $evictionHardMemory }}"
|
||||
nodefs.available: "10%"
|
||||
imagefs.available: "15%"
|
||||
nodefs.inodesFree: "5%"
|
||||
evictionSoft:
|
||||
memory.available: "{{ $evictionSoftMemory }}"
|
||||
nodefs.available: "15%"
|
||||
imagefs.available: "20%"
|
||||
evictionSoftGracePeriod:
|
||||
memory.available: "1m30s"
|
||||
nodefs.available: "1m30s"
|
||||
imagefs.available: "1m30s"
|
||||
evictionMinimumReclaim:
|
||||
memory.available: "256Mi"
|
||||
install:
|
||||
disk: /dev/vda
|
||||
image: factory.talos.dev/installer/{{ $.Values.talos.schematicID }}:{{ $talosVersion }}
|
||||
wipe: false
|
||||
features:
|
||||
rbac: true
|
||||
kubePrism:
|
||||
enabled: false
|
||||
cluster:
|
||||
id: ${CLUSTER_ID}
|
||||
secret: ${CLUSTER_SECRET}
|
||||
controlPlane:
|
||||
endpoint: https://${RELEASE}.${NS}.svc:6443
|
||||
clusterName: ${RELEASE}
|
||||
network:
|
||||
dnsDomain: {{ $dnsDomain }}
|
||||
podSubnets:
|
||||
- {{ $podCIDR }}
|
||||
serviceSubnets:
|
||||
- {{ $serviceCIDR }}
|
||||
token: ${BOOTSTRAP_TOKEN}
|
||||
ca:
|
||||
crt: ${K8S_CA_B64}
|
||||
discovery:
|
||||
enabled: true
|
||||
registries:
|
||||
kubernetes:
|
||||
disabled: true
|
||||
service:
|
||||
disabled: true
|
||||
EOF
|
||||
echo "[$(date -u +%H:%M:%S)] TalosConfigTemplate ${RELEASE}-${GROUP_NAME} applied"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
{{- end }}
|
||||
@@ -45,7 +45,7 @@
|
||||
],
|
||||
"properties": {
|
||||
"diskSize": {
|
||||
"description": "Persistent disk size for kubelet and containerd data.",
|
||||
"description": "System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.",
|
||||
"default": "20Gi",
|
||||
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
|
||||
"anyOf": [
|
||||
@@ -185,8 +185,7 @@
|
||||
"v1.34",
|
||||
"v1.33",
|
||||
"v1.32",
|
||||
"v1.31",
|
||||
"v1.30"
|
||||
"v1.31"
|
||||
]
|
||||
},
|
||||
"host": {
|
||||
@@ -898,12 +897,64 @@
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"properties": {
|
||||
"kubectl": {
|
||||
"description": "Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.",
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"talosCsrSigner": {
|
||||
"description": "Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.",
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"waitForKubeconfig": {
|
||||
"description": "Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.",
|
||||
"type": "string",
|
||||
"default": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"talos": {
|
||||
"description": "Talos worker image configuration.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"required": [
|
||||
"schematicID",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"schematicID": {
|
||||
"description": "Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.",
|
||||
"type": "string",
|
||||
"default": "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
},
|
||||
"version": {
|
||||
"description": "Talos release used for worker OS image and installer. Must satisfy the chart's Talos\u003c-\u003eKubernetes support matrix against the chosen `version`.",
|
||||
"type": "string",
|
||||
"default": "v1.13.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nodeHealthCheck": {
|
||||
"description": "MachineHealthCheck tuning for worker node groups.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"required": [
|
||||
"maxUnhealthy",
|
||||
"nodeStartupTimeout"
|
||||
],
|
||||
"properties": {
|
||||
"maxUnhealthy": {
|
||||
"description": "Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default \"50%\" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to \"0%\" once the fleet is stable on Talos workers.",
|
||||
"type": "string",
|
||||
"default": "50%"
|
||||
},
|
||||
"nodeStartupTimeout": {
|
||||
"description": "Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).",
|
||||
"type": "string",
|
||||
"default": "10m"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,7 +81,7 @@ storageClass: replicated
|
||||
## @field {int} maxReplicas=10 - Maximum number of replicas.
|
||||
## @field {string} instanceType="u1.medium" - Virtual machine instance type.
|
||||
## @x-cozystack-options {source: instancetype}
|
||||
## @field {quantity} diskSize="20Gi" - Persistent disk size for kubelet and containerd data.
|
||||
## @field {quantity} diskSize="20Gi" - System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.
|
||||
## @field {string} [storageClass] - StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.
|
||||
## @x-cozystack-options {source: storageclass}
|
||||
## @field {[]string} roles - List of node roles.
|
||||
@@ -110,7 +110,6 @@ nodeGroups:
|
||||
## @value v1.33
|
||||
## @value v1.32
|
||||
## @value v1.31
|
||||
## @value v1.30
|
||||
|
||||
## @param {Version} version - Kubernetes major.minor version to deploy
|
||||
version: "v1.35"
|
||||
@@ -273,7 +272,37 @@ controlPlane:
|
||||
|
||||
## @typedef {struct} Images - Optional image overrides for chart-internal helpers.
|
||||
## @field {string} [waitForKubeconfig] - Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.
|
||||
## @field {string} [kubectl] - Image used by the bootstrap-token tenant Job (kubectl). Empty falls back to images/kubectl.tag.
|
||||
## @field {string} [talosCsrSigner] - Image used by the talos-csr-signer sidecar in the Kamaji control plane. Empty falls back to images/talos-csr-signer.tag.
|
||||
|
||||
## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries.
|
||||
images:
|
||||
waitForKubeconfig: ""
|
||||
kubectl: ""
|
||||
talosCsrSigner: ""
|
||||
|
||||
##
|
||||
## @section Talos Worker Image
|
||||
##
|
||||
|
||||
## @typedef {struct} Talos - Talos worker image configuration.
|
||||
## @field {string} version - Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.
|
||||
## @field {string} schematicID - Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.
|
||||
|
||||
## @param {Talos} talos - Talos worker image configuration.
|
||||
talos:
|
||||
version: "v1.13.0"
|
||||
schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
|
||||
##
|
||||
## @section Node Health Check
|
||||
##
|
||||
|
||||
## @typedef {struct} NodeHealthCheck - MachineHealthCheck configuration for worker node groups.
|
||||
## @field {string} maxUnhealthy="50%" - Maximum number of unhealthy nodes tolerated per node group before remediation is paused. The MHC admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is to express the value as a percentage. Default "50%" leaves headroom for transient unhealthy nodes during the kubeadm-to-Talos rollover and slow first boots from factory.talos.dev. Drop to "0%" once the fleet is stable on Talos workers.
|
||||
## @field {string} nodeStartupTimeout="10m" - Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator).
|
||||
|
||||
## @param {NodeHealthCheck} nodeHealthCheck - MachineHealthCheck tuning for worker node groups.
|
||||
nodeHealthCheck:
|
||||
maxUnhealthy: "50%"
|
||||
nodeStartupTimeout: "10m"
|
||||
@@ -15,14 +15,24 @@ metadata:
|
||||
# deadline chosen to stay strictly below this value so the pod's
|
||||
# CrashLoopBackOff surfaces before flux remediation fires. If this
|
||||
# annotation is raised, update that init deadline correspondingly.
|
||||
release.cozystack.io/helm-install-timeout: "15m"
|
||||
release.cozystack.io/helm-install-timeout: "20m"
|
||||
# The kubernetes chart emits in-tenant addon HelmReleases (cilium, coredns,
|
||||
# csi, ingress-nginx, metrics-server, ouroboros) and ships a talos-reconcile
|
||||
# post-install hook that produces the TalosConfigTemplate the worker
|
||||
# MachineSets clone from. Without DisableWait the helm-controller waits for
|
||||
# the addon HelmReleases to be Ready before running the hook, but those
|
||||
# cannot be Ready until worker nodes exist, and the workers cannot be
|
||||
# created without the TalosConfigTemplate the hook is supposed to produce.
|
||||
# Skip the readiness wait so the hook fires and the chicken-and-egg
|
||||
# resolves.
|
||||
release.cozystack.io/helm-install-disable-wait: "true"
|
||||
spec:
|
||||
application:
|
||||
kind: Kubernetes
|
||||
singular: kubernetes
|
||||
plural: kuberneteses
|
||||
openAPISchema: |-
|
||||
{"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.Line truncated
|
||||
{"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image (factory.talos.dev raw artifact streamed in by CDI), kubelet state, containerd image cache, and any local-path PVCs. Pre-Talos installs used a separate disk-kubelet PVC for kubelet/containerd state; on Talos this is consolidated onto the single system disk imaged from the factory artifact.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ciliuLine truncated
|
||||
release:
|
||||
prefix: kubernetes-
|
||||
labels:
|
||||
@@ -40,7 +50,7 @@ spec:
|
||||
tags:
|
||||
- compute
|
||||
icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MLine truncated
|
||||
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "diskSize"], ["spec", "nodeGroups", "md0", "storageClass"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "nodeGroups", "md0", "kubelet"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]]
|
||||
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "diskSize"], ["spec", "nodeGroups", "md0", "storageClass"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "nodeGroups", "md0", "kubelet"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"], ["spec", "images", "kubectl"], ["spec", "images", "talosCsrSigner"], ["spec", "talos"], ["spec", "talos", "version"], ["spec", "talos", "schematicID"], ["spec", "nodeHealthCheck"], ["spec", "nodeHealthCheck", "maxUnhealthy"], ["spec", "nodeHealthCheck", "nodeStartupTimeout"]]
|
||||
secrets:
|
||||
exclude: []
|
||||
include:
|
||||
|
||||
Reference in new issue
Block a user