From 26dc0778096a92032b7a50b0b728f6d620825f82 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:08:47 +0300 Subject: [PATCH 1/6] fix(vm-disk): refuse cloning a golden whose importer keeps failing CDI moves an import to Failed only on a fatal error such as a checksum mismatch. An unreachable URL restarts the importer with back-off while the DataVolume stays in progress, so the Failed check let such a golden through and the clone waited forever. Treat an importer that restarted at least three times as failing too, whether or not it is running now. CDI reports every importer attempt as running, including one that is about to fail again, and the chart checks the golden only until the disk exists, so a disk created during an attempt would otherwise pass and wait unnoticed. A healthy import with three early restarts is refused until it succeeds; the release retries the render. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- packages/apps/vm-disk/templates/dv.yaml | 19 ++- .../vm-disk/tests/dv_golden_image_test.yaml | 147 +++++++++++++++++- .../tests/dv_golden_storageclass_test.yaml | 42 +++++ 3 files changed, 200 insertions(+), 8 deletions(-) diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml index c8750afe1..ece50fa0a 100644 --- a/packages/apps/vm-disk/templates/dv.yaml +++ b/packages/apps/vm-disk/templates/dv.yaml @@ -12,8 +12,17 @@ {{- if hasKey .Values.source "image" }} {{- $desiredSource = dict "pvc" (dict "name" (printf "vm-default-images-%s" (required "A valid .Values.source.image.name entry required!" .Values.source.image.name)) "namespace" "cozy-public") }} {{- /* Refuse a golden that is missing or whose import Failed: CDI never - populates it, so the clone would wait forever. A golden still importing - is left alone because CDI holds the clone until the source is populated. + populates it, so the clone would wait forever. Failed is reached only on + a fatal error such as a checksum mismatch; an unreachable URL instead + restarts the importer with back-off while the phase stays in progress, + so an importer that restarted at least three times counts as failing + too, running or not: CDI reports every attempt as running, including + one that is about to fail again, and this check runs only until the + disk exists, so a disk created during an attempt would pass and wait + unnoticed. A healthy import with three early restarts is refused until + it succeeds, and the release retries the render. Any other golden still + importing is left alone because CDI holds the clone until the source is + populated. Missing means neither the DataVolume nor the PVC CDI clones exists, as the DataVolume can precede its PVC, and counts only when the release namespace is visible: helm template has no cluster and every lookup @@ -24,8 +33,12 @@ {{- if not (or $existingDV (lookup "v1" "PersistentVolumeClaim" .Release.Namespace .Release.Name)) }} {{- $golden := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" "cozy-public" $desiredSource.pvc.name }} {{- $goldenPVC := lookup "v1" "PersistentVolumeClaim" "cozy-public" $desiredSource.pvc.name }} - {{- if eq (dig "status" "phase" "" $golden | toString) "Failed" }} + {{- $goldenPhase := dig "status" "phase" "" $golden | toString }} + {{- $goldenRestarts := dig "status" "restartCount" 0 $golden | int }} + {{- if eq $goldenPhase "Failed" }} {{- fail (printf "source.image %q: golden DataVolume %q in cozy-public failed to import (phase Failed), so a clone of it would never complete. Ask the platform administrator to check the importer pod in cozy-public, then delete that DataVolume to retry the import." .Values.source.image.name $desiredSource.pvc.name) }} + {{- else if and (ne $goldenPhase "Succeeded") (ge $goldenRestarts 3) }} + {{- fail (printf "source.image %q: golden DataVolume %q in cozy-public has not finished importing after %d importer restarts, so a clone of it would not complete while the import keeps failing. The refusal lifts by itself once the import succeeds. The importer is retried with a growing back-off, so after the source is fixed the refusal lasts for the current back-off delay plus the full image download. Ask the platform administrator to check the importer pod logs in cozy-public." .Values.source.image.name $desiredSource.pvc.name $goldenRestarts) }} {{- else if and (not $golden) (not $goldenPVC) (lookup "v1" "Namespace" "" .Release.Namespace) }} {{- fail (printf "source.image %q: golden image %q not found in cozy-public, so a clone of it would never start. Check the image name, or ask the platform administrator to add cozystack.vm-default-images to bundles.enabledPackages." .Values.source.image.name $desiredSource.pvc.name) }} {{- end }} diff --git a/packages/apps/vm-disk/tests/dv_golden_image_test.yaml b/packages/apps/vm-disk/tests/dv_golden_image_test.yaml index d901c6c96..08af3c406 100644 --- a/packages/apps/vm-disk/tests/dv_golden_image_test.yaml +++ b/packages/apps/vm-disk/tests/dv_golden_image_test.yaml @@ -1,8 +1,9 @@ -suite: vm-disk refuses to clone a golden image that is missing or failed -# A golden that is missing or whose import Failed is never populated, and CDI -# holds a clone of it forever, so a new disk on it would sit pending with -# nothing pointing at the image. A golden still importing is safe to clone -# because CDI waits for it, and helm template has no cluster to look it up in. +suite: vm-disk refuses to clone a golden image that is missing, failed or stuck +# A golden that is missing, whose import Failed, or whose importer keeps +# crashing is not populated, and CDI holds a clone of it until it is, so a new +# disk on it would sit pending with nothing pointing at the image. A golden +# still importing is safe to clone because CDI waits for it, and helm template +# has no cluster to look it up in. release: name: test-vm-disk namespace: tenant-test @@ -185,3 +186,139 @@ tests: - equal: path: spec.source.pvc value: {name: vm-default-images-ubuntu, namespace: cozy-public} + + - it: refuses a new disk on a golden whose importer keeps failing + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: &stuck + phase: ImportInProgress + # A string: helm-unittest's lookup cannot deep-copy an int, and | int takes either. + restartCount: "3" + conditions: + - {type: Bound, status: "True"} + - {type: Running, status: "False", reason: Error} + asserts: + - failedTemplate: + errorPattern: 'golden DataVolume "vm-default-images-ubuntu" in cozy-public has not finished importing after 3 importer restarts.*lifts by itself once the import succeeds' + + # CDI reports Running=True for every importer attempt, including one that is + # about to fail again, and this check runs only until the disk exists. + - it: refuses a new disk on a golden that restarted and is running again + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: {name: tenant-test} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: + phase: ImportInProgress + restartCount: "5" + conditions: + - {type: Running, status: "True", reason: Pod is running} + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + asserts: + - failedTemplate: + errorPattern: 'has not finished importing after 5 importer restarts' + + - it: clones a golden whose importer failed fewer than three times + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: {name: tenant-test} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: + phase: ImportInProgress + restartCount: "2" + conditions: + - {type: Running, status: "False", reason: Error} + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + asserts: + - notFailedTemplate: {} + + - it: clones a golden that Succeeded after its importer restarted + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: {name: tenant-test} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: + phase: Succeeded + restartCount: "4" + conditions: + - {type: Running, status: "False", reason: Completed} + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + asserts: + - notFailedTemplate: {} + + - it: leaves an existing disk alone when its golden is stuck + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: *stuck + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: test-vm-disk + namespace: tenant-test + annotations: + vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}' + vm-disk.cozystack.io/storage-class: replicated + spec: + source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}} + storage: {resources: {requests: {storage: 5Gi}}, storageClassName: replicated} + asserts: + - notFailedTemplate: {} + + - it: refuses a new disk on a golden whose importer restarted and reports no Running condition + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: {name: tenant-test} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: {phase: ImportInProgress, restartCount: "3"} + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + asserts: + - failedTemplate: + errorPattern: 'has not finished importing after 3 importer restarts' diff --git a/packages/apps/vm-disk/tests/dv_golden_storageclass_test.yaml b/packages/apps/vm-disk/tests/dv_golden_storageclass_test.yaml index 48b7dbf2e..a5da84d41 100644 --- a/packages/apps/vm-disk/tests/dv_golden_storageclass_test.yaml +++ b/packages/apps/vm-disk/tests/dv_golden_storageclass_test.yaml @@ -1115,6 +1115,48 @@ tests: asserts: - notFailedTemplate: {} + - it: leaves a disk alone whose PVC exists without its DataVolume when the golden is stuck + set: + source: {image: {name: ubuntu}} + storageClass: local + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *diskPVC + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: &stuck + phase: ImportInProgress + restartCount: "3" + conditions: + - {type: Running, status: "False", reason: Error} + asserts: + - notFailedTemplate: {} + + # Fixing the class alone would still leave the clone waiting on the import. + - it: names the stuck import, not the class, for a disk that trips both + set: + source: {image: {name: ubuntu}} + storageClass: local + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - *cdiCSIClone + - *replicatedClass + - *localClass + - *localProfile + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: *stuck + asserts: + - failedTemplate: + errorPattern: 'golden DataVolume "vm-default-images-ubuntu" in cozy-public has not finished importing after 3 importer restarts' + # A key under another prefix belongs to someone else and is not read. - it: clones into a LINSTOR class that allows remote access next to a foreign key that denies it set: From 513d250a68b1bbffd16f0504994ca245690537c7 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:10:08 +0300 Subject: [PATCH 2/6] fix(kubernetes-nodes): refuse a golden whose importer keeps failing The worker image guard refused only a golden in phase Failed, which CDI reaches on a fatal error such as a checksum mismatch. An unreachable URL restarts the importer with back-off while the DataVolume stays in progress, so a pool pinned to such a golden waited forever for its first disk. Refuse an importer that restarted at least three times and is not running, under the same already-cloned exemption as Failed. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../kubernetes-nodes/templates/nodegroup.yaml | 19 +++- .../tests/worker_image_cloned_pool_test.yaml | 36 ++++++- .../tests/worker_image_source_test.yaml | 96 +++++++++++++++++++ .../kubernetes-worker-image/values.yaml | 8 +- 4 files changed, 151 insertions(+), 8 deletions(-) diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 440bbfa05..bbf8c24d8 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -302,7 +302,7 @@ spec: and no CR edit is needed to trigger the abort, only the next periodic reconcile. The comment below rejects exactly that blast radius for a mid-import golden; applying the same reasoning to a golden that is - absent or Failed is what it costs to be consistent. + absent or failing to import is what it costs to be consistent. So the loud failure stays where it is the only signal — a pool still waiting for its first disk — and a pool that is past that point is @@ -367,21 +367,32 @@ spec: {{- fail (printf "nodeGroup %q: osImage.builtin requested golden image %q not found in cozy-public, neither as a DataVolume nor as a PVC — enable the cozystack.kubernetes-worker-image catalog via bundles.enabledPackages with a matching (schematicID, version) entry, or switch this pool to osImage.factory." .groupName $goldenName) }} {{- end }} {{- else }} - {{- /* Fail only on the golden's terminal failure phase, not on "not yet + {{- /* Fail only on a golden whose import is failing, not on "not yet Succeeded". A golden mid-import is transient: CDI holds the worker DataVolume until the source is populated, which is the correct behaviour and recovers unattended. Hard-failing the render there would abort the ENTIRE KubernetesNodes HelmRelease — every resource, not just this disk — for every pool pinned to that golden, blocking unrelated - scale operations for the length of an import. Only Failed, which - never recovers without operator action, justifies that blast radius. + scale operations for the length of an import. Failed never recovers + without operator action, but CDI reaches it only on a fatal error such + as a checksum mismatch: an unreachable URL restarts the importer with + back-off while the phase stays in progress. So an importer that + restarted at least three times and is not running now is refused too; + that refusal lifts on its own once an attempt succeeds. Falling back to the HTTP import is deliberately not an option either: it would make disk-system's source depend on cluster state, so the source — and with it the content-hashed KubevirtMachineTemplate name — would flip once the golden went Ready and roll every worker in the pool. */}} {{- $goldenPhase := dig "status" "phase" "" $goldenDV | toString }} + {{- $goldenRestarts := dig "status" "restartCount" 0 $goldenDV | int }} + {{- $goldenRunning := "" }} + {{- range (dig "status" "conditions" (list) $goldenDV) }} + {{- if eq (dig "type" "" . | toString) "Running" }}{{- $goldenRunning = dig "status" "" . | toString }}{{- end }} + {{- end }} {{- if and (eq $goldenPhase "Failed") (not $poolCloned) }} {{- fail (printf "nodeGroup %q: golden DataVolume %q in cozy-public failed to import (phase Failed) — cloning it would block this pool's workers indefinitely. Check the importer pod in cozy-public, then delete the DataVolume to retry the import." .groupName $goldenName) }} + {{- else if and (ne $goldenPhase "Succeeded") (ge $goldenRestarts 3) (eq $goldenRunning "False") (not $poolCloned) }} + {{- fail (printf "nodeGroup %q: golden DataVolume %q in cozy-public keeps failing to import (%d importer restarts) — cloning it would block this pool's workers while it does. The refusal lifts by itself while a new import attempt is running, and for good once one succeeds. The importer is retried with a growing back-off, and a recreated importer pod counts as not running until its container starts, so after the source is fixed the refusal lasts at least until the current back-off delay ends and the next importer starts. Check the importer pod logs in cozy-public." .groupName $goldenName $goldenRestarts) }} {{- end }} {{- /* CDI takes the storage-layer CSI-clone path only when source and target share a StorageClass; across classes it silently falls back to a diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml index 1bc4adb8a..8467ae6df 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml @@ -1,6 +1,6 @@ suite: osImage.builtin golden guards on a pool that has already cloned -# The absence and Failed guards abort the ENTIRE KubernetesNodes HelmRelease, and +# The absence and failed-import guards abort the ENTIRE KubernetesNodes HelmRelease, and # for a pool whose workers all cloned long ago that price buys nothing: replica # changes, kubelet retuning and MachineHealthCheck timeouts have no stake in the # golden, and no CR edit is needed to trigger the abort — the next periodic @@ -152,6 +152,30 @@ kubernetesProvider: resources: {requests: {storage: 6Gi}} storageClassName: replicated status: {phase: Failed} + # md0's own disk against a golden whose re-import is now stuck: the pool + # already holds its copy, so it needs nothing from the source. + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: disk-system-kubernetes-myk8s-md0-pqrst + namespace: tenant-test + annotations: + kubernetes-worker-image.cozystack.io/golden: talos-worker-stucklater-v1.0.0 + kubernetes-worker-image.cozystack.io/pool: kubernetes-myk8s-md0 + status: {phase: Succeeded} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-stucklater-v1.0.0, namespace: cozy-public} + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: replicated + status: + phase: ImportInProgress + # A string: helm-unittest's lookup cannot deep-copy an int, and | int takes either. + restartCount: "7" + conditions: + - {type: Running, status: "False", reason: Error} tests: - it: renders when the golden is gone but this pool already cloned it @@ -217,3 +241,13 @@ tests: asserts: - failedTemplate: errorPattern: 'golden DataVolume "talos-worker-sibling-v1\.0\.0" in cozy-public failed to import' + + - it: renders when the golden importer keeps failing but this pool already cloned it + set: + osImage: + builtin: {schematicID: stucklater, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-stucklater-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml index 2794e824b..b076f50bd 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml @@ -119,6 +119,54 @@ kubernetesProvider: storageClassName: replicated status: phase: Failed + # Goldens whose importer has restarted. CDI leaves an unreachable URL in + # progress and restarts the importer instead of reaching Failed, so restarts + # plus an importer that is not running now is what marks one as failing. + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-stuck-v1.0.0, namespace: cozy-public} + spec: &goldenSpec + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: replicated + status: + phase: ImportInProgress + # A string: helm-unittest's lookup cannot deep-copy an int, and | int takes either. + restartCount: "3" + conditions: + - {type: Running, status: "False", reason: Error} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-recovered-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: + phase: ImportInProgress + restartCount: "5" + conditions: + - {type: Running, status: "True", reason: Pod is running} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-flaky-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: + phase: ImportInProgress + restartCount: "2" + conditions: + - {type: Running, status: "False", reason: Error} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-done-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: + phase: Succeeded + restartCount: "4" + conditions: + - {type: Running, status: "False", reason: Completed} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-norunning-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: {phase: ImportInProgress, restartCount: "3"} # A populated golden that declares no storage request at all. The size guard # reads spec.storage.resources.requests.storage, so its presence gate is the # branch that decides between comparing and skipping; without a fixture the @@ -271,6 +319,54 @@ tests: - failedTemplate: errorPattern: 'golden DataVolume "talos-worker-failed-v1\.0\.0" in cozy-public failed to import \(phase Failed\)' + - it: osImage.builtin fails the render when the golden importer keeps failing + set: + osImage: + builtin: {schematicID: stuck, version: v1.0.0} + asserts: + - failedTemplate: + errorPattern: 'golden DataVolume "talos-worker-stuck-v1\.0\.0" in cozy-public keeps failing to import \(3 importer restarts\).*lifts by itself.*a recreated importer pod counts as not running until its container starts' + + - it: osImage.builtin clones a golden whose importer restarted but is running again + set: + osImage: + builtin: {schematicID: recovered, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-recovered-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: osImage.builtin clones a golden whose importer failed fewer than three times + set: + osImage: + builtin: {schematicID: flaky, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-flaky-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: osImage.builtin clones a golden that Succeeded after its importer restarted + set: + osImage: + builtin: {schematicID: done, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-done-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: osImage.builtin clones a golden whose importer restarted and reports no Running condition + set: + osImage: + builtin: {schematicID: norunning, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-norunning-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + # Cross-StorageClass clone: CDI would silently fall back to a host-assisted copy # over the pod network, the exact transfer osImage.builtin exists to avoid. - it: osImage.builtin fails when the pool storageClass differs from the golden's diff --git a/packages/system/kubernetes-worker-image/values.yaml b/packages/system/kubernetes-worker-image/values.yaml index 67fb268ba..169632ba6 100644 --- a/packages/system/kubernetes-worker-image/values.yaml +++ b/packages/system/kubernetes-worker-image/values.yaml @@ -30,10 +30,12 @@ ## `talos-worker--` from cozy-public. The tenant render fails ## rather than degrade silently when the referenced (schematicID, version) has no ## matching entry in this catalog, when its golden terminally failed to import -## (`phase: Failed`), or when the node group's storageClass differs from the +## (`phase: Failed`) or keeps failing to (its importer restarted at least 3 times +## and is not running), or when the node group's storageClass differs from the ## golden's — so keep the catalog in sync with the (schematicID, version) pairs and -## StorageClasses that tenant node groups reference. A golden that is merely -## mid-import is not an error: CDI holds the worker disk until the source populates. +## StorageClasses that tenant node groups reference. A golden still importing, with +## its importer running or restarted fewer than 3 times, is not an error: CDI holds +## the worker disk until the source populates. ## Goldens carry `helm.sh/resource-policy: keep`, so removing an `images[]` entry ## stops managing that golden rather than deleting it. Deleting one by hand while a ## worker pool still clones it fails that pool's render on its next values or chart From 263407a73f6a8e51ed677373f97d4790270b31f2 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:10:42 +0300 Subject: [PATCH 3/6] fix(api): leave golden images whose importer keeps failing out of the image options CDI moves an import to Failed only on a fatal error such as a checksum mismatch, so the picker still offered a golden whose URL is unreachable: its importer restarts with back-off while the DataVolume stays in progress. Drop a golden whose importer restarted at least three times and is not running, the test the kubernetes-nodes chart applies before cloning. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../system/cozystack-api/templates/rbac.yaml | 2 +- pkg/registry/core/option/providers.go | 36 ++++++++++++++++--- pkg/registry/core/option/providers_test.go | 35 ++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/packages/system/cozystack-api/templates/rbac.yaml b/packages/system/cozystack-api/templates/rbac.yaml index b5501b8cc..a1eba49eb 100644 --- a/packages/system/cozystack-api/templates/rbac.yaml +++ b/packages/system/cozystack-api/templates/rbac.yaml @@ -48,7 +48,7 @@ rules: - apiGroups: ["kubevirt.io"] resources: ["kubevirts"] verbs: ["get", "watch", "list"] -# The image picker drops golden images whose import Failed. +# The image picker drops golden images whose import failed or keeps failing. - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] verbs: ["get", "watch", "list"] diff --git a/pkg/registry/core/option/providers.go b/pkg/registry/core/option/providers.go index 7a9ab6e1c..a93007a1c 100644 --- a/pkg/registry/core/option/providers.go +++ b/pkg/registry/core/option/providers.go @@ -335,9 +335,10 @@ func permittedResourceNames(kv *unstructured.Unstructured) map[string]struct{} { // imageProvider lists the default image PVCs in cozy-public and strips the // vm-default-images- prefix to get the catalog name used by the vm-disk chart. // -// A golden whose DataVolume is in phase Failed is left out: CDI never -// populates it, so a disk cloned from it never becomes usable. One still -// importing stays, because CDI holds a clone until the source is populated. +// A golden whose import is failing is left out (see goldenImportFailing): +// CDI does not populate it, so a disk cloned from it never becomes usable. +// One still importing stays, because CDI holds a clone until the source is +// populated. func imageProvider(dyn dynamic.Interface) providerFunc { return func(ctx context.Context, _ string) ([]corev1alpha1.OptionItem, error) { list, err := dyn.Resource(gvrPVCs).Namespace(publicImagesNamespace).List(ctx, listOpts()) @@ -353,7 +354,7 @@ func imageProvider(dyn dynamic.Interface) providerFunc { logProviderError("image", err) } else { for i := range dvs.Items { - if phase, _, _ := unstructured.NestedString(dvs.Items[i].Object, "status", "phase"); phase == "Failed" { + if goldenImportFailing(&dvs.Items[i]) { failed[dvs.Items[i].GetName()] = true } } @@ -371,6 +372,33 @@ func imageProvider(dyn dynamic.Interface) providerFunc { } } +// goldenImportFailing reports a golden DataVolume whose import is failing. +// CDI sets phase Failed only on a fatal error such as a checksum mismatch; an +// unreachable URL restarts the importer with back-off while the phase stays +// in progress, so an importer that restarted at least three times and is not +// running now counts as failing too. The kubernetes-nodes chart applies the +// same test before cloning a golden; vm-disk refuses on the restarts alone. +func goldenImportFailing(dv *unstructured.Unstructured) bool { + phase, _, _ := unstructured.NestedString(dv.Object, "status", "phase") + if phase == "Failed" { + return true + } + if phase == "Succeeded" { + return false + } + restarts, _, _ := unstructured.NestedInt64(dv.Object, "status", "restartCount") + if restarts < 3 { + return false + } + conditions, _, _ := unstructured.NestedSlice(dv.Object, "status", "conditions") + for _, c := range conditions { + if cm, ok := c.(map[string]any); ok && cm["type"] == "Running" { + return cm["status"] == "False" + } + } + return false +} + // storagePoolProvider derives selectable pool names from the seaweedfs // HelmRelease(s) in the request namespace (volume.pools and // volume.zones..pools map keys). Best-effort: returns nil when no diff --git a/pkg/registry/core/option/providers_test.go b/pkg/registry/core/option/providers_test.go index 43f91a26b..1f5df4241 100644 --- a/pkg/registry/core/option/providers_test.go +++ b/pkg/registry/core/option/providers_test.go @@ -344,6 +344,41 @@ func TestImageProviderSkipsGoldensWhoseImportFailed(t *testing.T) { } } +func TestImageProviderSkipsGoldensWhoseImporterKeepsFailing(t *testing.T) { + pvcGVK := gvrPVCs.GroupVersion().WithKind("PersistentVolumeClaim") + dvGVK := gvrDataVolumes.GroupVersion().WithKind("DataVolume") + dv := func(name, phase string, restarts int64, running string) *unstructured.Unstructured { + o := newObj(dvGVK, publicImagesNamespace, name, nil) + _ = unstructured.SetNestedField(o.Object, phase, "status", "phase") + _ = unstructured.SetNestedField(o.Object, restarts, "status", "restartCount") + if running != "" { + _ = unstructured.SetNestedSlice(o.Object, []any{ + map[string]any{"type": "Bound", "status": "True"}, + map[string]any{"type": "Running", "status": running}, + }, "status", "conditions") + } + return o + } + golden := func(name string) *unstructured.Unstructured { + return newObj(pvcGVK, publicImagesNamespace, "vm-default-images-"+name, nil) + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), listKinds(), + golden("stuck"), golden("recovered"), golden("flaky"), golden("done"), golden("norunning"), + dv("vm-default-images-stuck", "ImportInProgress", 3, "False"), + dv("vm-default-images-recovered", "ImportInProgress", 5, "True"), + dv("vm-default-images-flaky", "ImportInProgress", 2, "False"), + dv("vm-default-images-done", "Succeeded", 4, "False"), + dv("vm-default-images-norunning", "ImportInProgress", 3, ""), + ) + items, err := DefaultProviders(dyn)["image"](context.Background(), "") + if err != nil { + t.Fatalf("image provider: %v", err) + } + if got := values(items); len(got) != 4 || got[0] != "done" || got[1] != "flaky" || got[2] != "norunning" || got[3] != "recovered" { + t.Fatalf("image: got %v, want [done flaky norunning recovered] (stuck golden dropped)", got) + } +} + func TestImageProviderListsGoldensWhenDataVolumesCannotBeRead(t *testing.T) { for name, listErr := range map[string]error{ "no CDI": apierrors.NewNotFound(gvrDataVolumes.GroupResource(), ""), From 8dcf950842cd6431d3c24adf075252a2b0c98333 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:12:41 +0300 Subject: [PATCH 4/6] fix(kubernetes-nodes)!: resolve an empty class to CDI's virtualization default CDI binds a kubevirt-content DataVolume that names no class on the class annotated storageclass.kubevirt.io/is-default-virt-class before it falls back to the Kubernetes default. The worker image guard resolved an empty class to the Kubernetes default only, so on a cluster carrying both it compared the golden against a class the golden was not on and refused a pool that matched it. CDI's fallback reads only the GA default annotation, so the beta one no longer ties with it: a class carrying only the beta annotation is picked only when no class carries the GA one. CDI then creates no claim at all for a DataVolume naming neither a class nor access modes, so such a pool gets that class only because the chart writes it onto the worker disk. The resolution also refuses a pool that rendered before: one with an empty class that has not cloned a disk yet, whose golden is on the Kubernetes default, on a cluster where another class carries the virtualization default. Its storageClass is immutable, so it has to point osImage.builtin at a catalog entry on the virtualization default or switch to osImage.factory. A pool that already cloned reads its class back from its own disk's claim and keeps rendering. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../kubernetes-nodes/templates/_helpers.tpl | 44 +++-- .../kubernetes-nodes/templates/nodegroup.yaml | 3 +- .../worker_image_beta_default_class_test.yaml | 81 ++++++++ ...worker_image_two_default_classes_test.yaml | 19 ++ .../worker_image_virt_default_class_test.yaml | 183 ++++++++++++++++++ 5 files changed, 313 insertions(+), 17 deletions(-) create mode 100644 packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml create mode 100644 packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml diff --git a/packages/apps/kubernetes-nodes/templates/_helpers.tpl b/packages/apps/kubernetes-nodes/templates/_helpers.tpl index 5e317f8e6..262b947c8 100644 --- a/packages/apps/kubernetes-nodes/templates/_helpers.tpl +++ b/packages/apps/kubernetes-nodes/templates/_helpers.tpl @@ -441,18 +441,25 @@ still fails. {{- end }} {{- /* -Name of the cluster's default StorageClass, or the empty string when there is -none (and always under `helm template`, which has no cluster to read). +Name of the StorageClass CDI binds a kubevirt-content DataVolume on when it +names no class, or the empty string when there is none (and always under `helm +template`, which has no cluster to read). CDI takes a class annotated +storageclass.kubevirt.io/is-default-virt-class first and only then the +Kubernetes default (GetStorageClassByNameWithVirtFallback in CDI's +pkg/controller/common/util.go), and both the goldens and the worker disks are +kubevirt content. An empty storageClass is a documented, schema-valid setting on both a worker pool and a worker image catalog entry, and it means "the cluster default". Without resolving it the golden-versus-pool StorageClass comparison simply skips whenever -either side is empty, which is the one corner where skipping is worst: CDI then -falls back to a host-assisted copy over the pod network, silently, and that copy -is the transfer the clone path exists to remove. +either side is empty, and lets a cross-class clone through unchecked. -Both the current annotation and its beta predecessor count, because clusters -provisioned years apart carry different ones and Kubernetes still honours both. +The beta predecessor of the Kubernetes annotation is read last, only when no +class carries the current one. CDI does not read it: with neither of the other +two defaults it creates no claim at all for a DataVolume that names no class +and no access modes (pvcFromStorage, ErrStorageClassNotFound). Read here, it +still gives a pool on such a cluster a class, written onto the worker disk +explicitly. More than one class may carry the annotation at once. Kubernetes permits that -- it is the normal state midway through swapping a cluster's default -- and @@ -463,16 +470,21 @@ that is actually in force. Timestamps are RFC3339 in UTC, so comparing them as strings is comparing them chronologically. */ -}} {{- define "kubernetes-nodes.defaultStorageClassName" -}} -{{- $classes := lookup "storage.k8s.io/v1" "StorageClass" "" "" -}} +{{- $classes := dig "items" (list) ((lookup "storage.k8s.io/v1" "StorageClass" "" "") | default dict) -}} {{- $name := "" -}} -{{- $createdAt := "" -}} -{{- range (dig "items" (list) ($classes | default dict)) -}} -{{- $annotations := dig "metadata" "annotations" (dict) . -}} -{{- if or (eq (dig "storageclass.kubernetes.io/is-default-class" "" $annotations | toString) "true") (eq (dig "storageclass.beta.kubernetes.io/is-default-class" "" $annotations | toString) "true") -}} -{{- $at := dig "metadata" "creationTimestamp" "" . | toString -}} -{{- if or (not $name) (gt $at $createdAt) -}} -{{- $name = dig "metadata" "name" "" . | toString -}} -{{- $createdAt = $at -}} +{{- range list "storageclass.kubevirt.io/is-default-virt-class" "storageclass.kubernetes.io/is-default-class" "storageclass.beta.kubernetes.io/is-default-class" -}} +{{- $key := . -}} +{{- $createdAt := "" -}} +{{- if not $name -}} +{{- range $classes -}} +{{- $annotations := dig "metadata" "annotations" (dict) . -}} +{{- if eq (dig $key "" $annotations | toString) "true" -}} +{{- $at := dig "metadata" "creationTimestamp" "" . | toString -}} +{{- if or (not $name) (gt $at $createdAt) -}} +{{- $name = dig "metadata" "name" "" . | toString -}} +{{- $createdAt = $at -}} +{{- end -}} +{{- end -}} {{- end -}} {{- end -}} {{- end -}} diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index bbf8c24d8..39d71b7be 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -540,7 +540,8 @@ spec: rejects RWX volumes that are not on a DRBD-backed StorageClass, which is why the pool's storageClass defaults to a replicated/DRBD class rather than to the cluster default. An empty storageClass - omits the field so the cluster default applies. + omits the field, and CDI binds the disk on its virtualization + default or, without one, the Kubernetes default. A builtin pool instead renders the class the guard above resolved for it. Omitting the field there would let the disk bind on whatever the diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml new file mode 100644 index 000000000..d1f7787e8 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml @@ -0,0 +1,81 @@ +suite: osImage.builtin default-StorageClass selection with only the beta annotation + +# CDI does not read the beta annotation, so the pool's class comes from the +# chart alone, written onto the worker disk explicitly. + +templates: + - templates/nodegroup.yaml + +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test + +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: [ingress-nginx] + osImage: + builtin: {} + +kubernetesProvider: + scheme: + "instancetype.kubevirt.io/v1beta1/VirtualMachineClusterInstancetype": + gvr: {group: instancetype.kubevirt.io, version: v1beta1, resource: virtualmachineclusterinstancetypes} + namespaced: false + "cluster.x-k8s.io/v1beta1/MachineSet": + gvr: {group: cluster.x-k8s.io, version: v1beta1, resource: machinesets} + namespaced: true + "infrastructure.cluster.x-k8s.io/v1alpha1/KubevirtMachineTemplate": + gvr: {group: infrastructure.cluster.x-k8s.io, version: v1alpha1, resource: kubevirtmachinetemplates} + namespaced: true + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes} + namespaced: true + "storage.k8s.io/v1/StorageClass": + gvr: {group: storage.k8s.io, version: v1, resource: storageclasses} + namespaced: false + objects: + - kind: VirtualMachineClusterInstancetype + apiVersion: instancetype.kubevirt.io/v1beta1 + metadata: {name: u1.medium} + - kind: MachineSet + apiVersion: cluster.x-k8s.io/v1beta1 + metadata: {name: ignored-ms, namespace: tenant-test} + spec: {template: {spec: {infrastructureRef: {kind: Other, name: ignored}}}} + - kind: KubevirtMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + metadata: {name: ignored-kmt, namespace: tenant-test} + # The outgoing default, created first. + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: beta-only + creationTimestamp: "2026-01-01T00:00:00Z" + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515-v1.13.6 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: beta-only + status: {phase: Succeeded} + +tests: + - it: resolves an empty pool storageClass to a class with only the beta annotation + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: beta-only + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml index 1c30a2d2c..5613c6302 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml @@ -114,3 +114,22 @@ tests: asserts: - failedTemplate: errorPattern: 'requested golden image "talos-worker-elsewhere-v1\.0\.0" not found' + + # CDI reads only the GA annotation when it picks a class itself, so a newer + # class carrying only the beta one does not displace a GA default. + - it: prefers a GA default over a newer class with only the beta annotation + kubernetesProvider: + objects: + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: beta-only + creationTimestamp: "2026-09-01T00:00:00Z" + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: new + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml new file mode 100644 index 000000000..d1b6f4652 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml @@ -0,0 +1,183 @@ +suite: osImage.builtin default-StorageClass selection with a virtualization default + +# CDI resolves a DataVolume with no class and contentType kubevirt to the class +# annotated storageclass.kubevirt.io/is-default-virt-class before it looks at the +# Kubernetes default, so a cluster that carries both binds the golden and the +# worker disks on the virtualization default. The clone guard has to resolve an +# empty class the same way, or it compares the golden against a class it is not +# on. + +templates: + - templates/nodegroup.yaml + +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test + +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: [ingress-nginx] + osImage: + builtin: {} + +kubernetesProvider: + scheme: + "instancetype.kubevirt.io/v1beta1/VirtualMachineClusterInstancetype": + gvr: {group: instancetype.kubevirt.io, version: v1beta1, resource: virtualmachineclusterinstancetypes} + namespaced: false + "cluster.x-k8s.io/v1beta1/MachineSet": + gvr: {group: cluster.x-k8s.io, version: v1beta1, resource: machinesets} + namespaced: true + "infrastructure.cluster.x-k8s.io/v1alpha1/KubevirtMachineTemplate": + gvr: {group: infrastructure.cluster.x-k8s.io, version: v1alpha1, resource: kubevirtmachinetemplates} + namespaced: true + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes} + namespaced: true + "storage.k8s.io/v1/StorageClass": + gvr: {group: storage.k8s.io, version: v1, resource: storageclasses} + namespaced: false + "v1/PersistentVolumeClaim": + gvr: {group: "", version: v1, resource: persistentvolumeclaims} + namespaced: true + objects: + - kind: VirtualMachineClusterInstancetype + apiVersion: instancetype.kubevirt.io/v1beta1 + metadata: {name: u1.medium} + - kind: MachineSet + apiVersion: cluster.x-k8s.io/v1beta1 + metadata: {name: ignored-ms, namespace: tenant-test} + spec: {template: {spec: {infrastructureRef: {kind: Other, name: ignored}}}} + - kind: KubevirtMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + metadata: {name: ignored-kmt, namespace: tenant-test} + # The Kubernetes default, created after the virtualization default so that + # creation order alone would pick it. + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: kube-default + creationTimestamp: "2026-06-01T00:00:00Z" + annotations: + storageclass.kubernetes.io/is-default-class: "true" + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: virt-default + creationTimestamp: "2026-01-01T00:00:00Z" + annotations: + storageclass.kubevirt.io/is-default-virt-class: "true" + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515-v1.13.6 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: virt-default + status: {phase: Succeeded} + +tests: + - it: resolves an empty pool storageClass to the virtualization default + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: virt-default + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: takes the most recently created of two virtualization defaults + kubernetesProvider: + objects: + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: virt-newer + creationTimestamp: "2026-03-01T00:00:00Z" + annotations: + storageclass.kubevirt.io/is-default-virt-class: "true" + asserts: + - failedTemplate: + errorPattern: 'storageClass "virt-newer" differs from golden image .* storageClass "virt-default".* An empty storageClass was resolved to the cluster default StorageClass "virt-newer"' + + - it: ignores a virtualization default annotation that is not "true" + kubernetesProvider: + objects: + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: virt-off + creationTimestamp: "2026-05-01T00:00:00Z" + annotations: + storageclass.kubevirt.io/is-default-virt-class: "false" + asserts: + - notFailedTemplate: {} + + # Before the virtualization default was read, a pool that has not cloned a + # disk yet resolved to the Kubernetes default here and rendered. + - it: refuses a pool whose golden is on the Kubernetes default instead + set: + osImage: + builtin: {schematicID: kube, version: v1.0.0} + kubernetesProvider: + objects: + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-kube-v1.0.0 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: kube-default + status: {phase: Succeeded} + asserts: + - failedTemplate: + errorPattern: 'storageClass "virt-default" differs from golden image .* storageClass "kube-default".* An empty storageClass was resolved to "virt-default"' + + # A pool that already cloned reads its class back from its own disk's claim + # and never reaches the default lookup. + - it: keeps rendering a pool that already cloned from a golden on the Kubernetes default + set: + osImage: + builtin: {schematicID: kube, version: v1.0.0} + kubernetesProvider: + objects: + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-kube-v1.0.0 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: kube-default + status: {phase: Succeeded} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: disk-system-kubernetes-myk8s-md0-abcde + namespace: tenant-test + annotations: + kubernetes-worker-image.cozystack.io/golden: talos-worker-kube-v1.0.0 + kubernetes-worker-image.cozystack.io/pool: kubernetes-myk8s-md0 + status: {phase: Succeeded} + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: disk-system-kubernetes-myk8s-md0-abcde, namespace: tenant-test} + spec: {storageClassName: kube-default} + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: kube-default + documentSelector: {path: kind, value: KubevirtMachineTemplate} From b6ef231d4496fa678f454027c959f789541a1134 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:14:34 +0300 Subject: [PATCH 5/6] docs(kubernetes-nodes): state why a pool must share its golden's class The worker image guard, its error message and the storageClass docs said CDI cannot clone across StorageClasses and always falls back to a host-assisted copy. CDI falls back only when the two claims have different CSI drivers or volume modes; with one driver it asks that driver for a clone into the other class. The guard stays, the reason it gives is corrected, and the docs name the virtualization default class an empty storageClass now resolves to. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- api/apps/v1alpha1/kubernetesnodes/types.go | 2 +- hack/check-worker-image-catalog-defaults.bats | 12 ++--- packages/apps/kubernetes-nodes/README.md | 6 +-- .../kubernetes-nodes/templates/nodegroup.yaml | 45 ++++++++++--------- ...orker_image_default_storageclass_test.yaml | 9 ++-- ...worker_image_frozen_storageclass_test.yaml | 6 +-- .../tests/worker_image_source_test.yaml | 4 +- .../worker_image_virt_default_class_test.yaml | 2 +- .../apps/kubernetes-nodes/values.schema.json | 2 +- packages/apps/kubernetes-nodes/values.yaml | 2 +- packages/core/platform/values.yaml | 8 ++-- .../cozyrds/kubernetes-nodes.yaml | 2 +- .../kubernetes-worker-image/values.yaml | 2 +- packages/system/vm-default-images/values.yaml | 2 +- 14 files changed, 54 insertions(+), 50 deletions(-) diff --git a/api/apps/v1alpha1/kubernetesnodes/types.go b/api/apps/v1alpha1/kubernetesnodes/types.go index 202807bb1..a8bb94b1e 100644 --- a/api/apps/v1alpha1/kubernetesnodes/types.go +++ b/api/apps/v1alpha1/kubernetesnodes/types.go @@ -22,7 +22,7 @@ type ConfigSpec struct { // +kubebuilder:default:="" // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="cluster is immutable" Cluster string `json:"cluster"` - // StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. + // StorageClass for the worker node system disk. When empty, CDI picks the class: the one annotated `storageclass.kubevirt.io/is-default-virt-class`, else the cluster default. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: the chart refuses a pool whose class differs from its golden's, because across classes CDI either falls back to a host-assisted copy over the pod network (different CSI drivers or volume modes) or leaves a clone into the other class to the CSI driver. Leaving it empty makes the same comparison against the class CDI would pick. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. // +kubebuilder:default:="replicated" // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="storageClass is immutable" StorageClass string `json:"storageClass"` diff --git a/hack/check-worker-image-catalog-defaults.bats b/hack/check-worker-image-catalog-defaults.bats index eaf942112..9b27a8572 100644 --- a/hack/check-worker-image-catalog-defaults.bats +++ b/hack/check-worker-image-catalog-defaults.bats @@ -13,10 +13,10 @@ # silently breaks every fresh install using osImage.builtin, and nothing catches it # until a ~95-minute e2e run does. # -# The StorageClass is the same trap from the other direction: CDI cannot -# CSI-clone across StorageClasses (it would silently fall back to a host-assisted -# copy over the pod network), so the render also rejects a group whose -# storageClass differs from its golden's. Bumping one file's default alone brings +# The StorageClass is the same trap from the other direction: the render also +# rejects a group whose storageClass differs from its golden's, because across +# classes CDI either falls back to a host-assisted copy over the pod network or +# leaves the clone to the CSI driver. Bumping one file's default alone brings # that guard down on the default path. # # Both checks are cheap and exact, which is the point — they turn a slow, remote @@ -70,8 +70,8 @@ CATALOG_VALUES="$REPO_ROOT/packages/system/kubernetes-worker-image/values.yaml" echo "StorageClass mismatch on the default osImage.builtin path:" >&2 echo " packages/apps/kubernetes-nodes/values.yaml storageClass: $app_sc" >&2 echo " kubernetes-worker-image golden effective storageClass: $entry_sc" >&2 - echo "CDI cannot CSI-clone across StorageClasses, so the tenant render" >&2 - echo "rejects this outright. Keep the two defaults in step." >&2 + echo "The tenant render refuses a pool whose storageClass differs from" >&2 + echo "its golden's. Keep the two defaults in step." >&2 return 1 fi } diff --git a/packages/apps/kubernetes-nodes/README.md b/packages/apps/kubernetes-nodes/README.md index ff1dd2b40..ef1c1875b 100644 --- a/packages/apps/kubernetes-nodes/README.md +++ b/packages/apps/kubernetes-nodes/README.md @@ -23,9 +23,9 @@ The pool's Kubernetes minor (`version`) may lag the cluster's during an upgrade ### Common Parameters -| Name | Description | Type | Value | -| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------ | -| `storageClass` | StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. | `string` | `replicated` | +| Name | Description | Type | Value | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------ | +| `storageClass` | StorageClass for the worker node system disk. When empty, CDI picks the class: the one annotated `storageclass.kubevirt.io/is-default-virt-class`, else the cluster default. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: the chart refuses a pool whose class differs from its golden's, because across classes CDI either falls back to a host-assisted copy over the pod network (different CSI drivers or volume modes) or leaves a clone into the other class to the CSI driver. Leaving it empty makes the same comparison against the class CDI would pick. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. | `string` | `replicated` | ### Pool Sizing diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 39d71b7be..5bf3eaa51 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -269,7 +269,7 @@ spec: {{- $httpURL := printf "%s/image/%s/%s/openstack-amd64.raw.xz" ($reqFactoryURL | trimSuffix "/") $reqSchematicID $reqVersion }} {{- $goldenName := printf "talos-worker-%s-%s" $reqSchematicID $reqVersion }} {{- /* A builtin pool whose golden is absent, failed to import, or on a - StorageClass the worker disk cannot CSI-clone from fails loudly here + StorageClass other than the worker disk's fails loudly here rather than degrading silently. And because a CDI clone target must be at least as large as its source, a worker diskSize below the golden's storage would fail the clone at runtime — reject it at render when the @@ -334,7 +334,7 @@ spec: from it too, and the two disagreeing is how a pool at storageClass "" passed the comparison on the class its existing disks are bound to and then bound its next disk on whatever the cluster default had become by - then -- taking the host-assisted copy the guard exists to refuse, with + then -- taking the cross-class clone the guard exists to refuse, with the guard reporting no drift. A pool that declares no class took the default in force when ITS disks @@ -394,19 +394,24 @@ spec: {{- else if and (ne $goldenPhase "Succeeded") (ge $goldenRestarts 3) (eq $goldenRunning "False") (not $poolCloned) }} {{- fail (printf "nodeGroup %q: golden DataVolume %q in cozy-public keeps failing to import (%d importer restarts) — cloning it would block this pool's workers while it does. The refusal lifts by itself while a new import attempt is running, and for good once one succeeds. The importer is retried with a growing back-off, and a recreated importer pod counts as not running until its container starts, so after the source is fixed the refusal lasts at least until the current back-off delay ends and the next importer starts. Check the importer pod logs in cozy-public." .groupName $goldenName $goldenRestarts) }} {{- end }} - {{- /* CDI takes the storage-layer CSI-clone path only when source and target - share a StorageClass; across classes it silently falls back to a + {{- /* Keep a pool on its golden's StorageClass. CDI falls back to a host-assisted copy over the pod network — exactly the per-worker - network transfer osImage.builtin exists to avoid (#3231). Reject the - case we can prove. An empty class on either side means "the cluster - default", which is resolvable against a live cluster and is resolved - here: leaving it unresolved is what let the one corner that CANNOT be - a clone — a pool at storageClass "" whose golden is on an explicit - class — pass unchecked. With no cluster to read (helm template) both - stay empty and the comparison is skipped, as before. This rules out - the guaranteed-host-assisted case, not every one: a matching - StorageClass still leaves the strategy to the CDI StorageProfile, - which the worker DataVolume inherits. */}} + network transfer osImage.builtin exists to avoid (#3231) — when the + two claims have different CSI drivers or volume modes + (validateAdvancedClonePVC in CDI's pkg/controller/clone/planner.go). + Two classes of one driver do not fall back: CDI asks that driver for + a snapshot or CSI clone into the other class, and whether that works + depends on the driver and on both classes' parameters. Sharing a + class rules out both a driver mismatch and a cross-class clone. An + empty class on either side means the class CDI binds on, which is + resolvable against a live cluster and is resolved here: leaving it + unresolved is what let a pool at storageClass "" whose golden is on + an explicit class pass unchecked. With no cluster to read (helm template) both stay empty + and the comparison is skipped, as before. A matching StorageClass + does not rule out every host-assisted copy: the strategy still comes + from the CDI StorageProfile the worker DataVolume inherits, and CDI + also falls back on differing volume modes, when no VolumeSnapshotClass + fits, or when a larger target cannot be expanded. */}} {{- $workerSC := $cloneWorkerSC }} {{- $goldenSC := dig "spec" "storage" "storageClassName" "" $goldenDV | toString }} {{- /* A golden that declares no class is not on "the cluster default" as it @@ -438,9 +443,9 @@ spec: made, and reports it as the empty string. */}} {{- $defaultNote := "" }} {{- if $resolvedFromDefault }} - {{- $defaultNote = printf " An empty storageClass was resolved to the cluster default StorageClass %q." $defaultSC }} + {{- $defaultNote = printf " An empty storageClass was resolved to %q, the class CDI binds a disk that names none on." $defaultSC }} {{- end }} - {{- fail (printf "nodeGroup %q: storageClass %q differs from golden image %q storageClass %q — CDI cannot CSI-clone across StorageClasses and would fall back to a host-assisted copy over the pod network.%s Neither side can be moved in place: this golden's class is fixed for its lifetime, because a DataVolume spec cannot be patched, and the pool's storageClass is immutable on an existing pool. So add a NEW kubernetes-worker-image catalog entry with a different (schematicID, version) on storageClass %q (kubernetesWorkerImage.images in the cozystack-platform Package values) and point this pool's osImage.builtin at it; or, once no pool clones this golden, delete it and let the catalog recreate it on that class. Switching this pool to osImage.factory also resolves it." .groupName $workerSC $goldenName $goldenSC $defaultNote $workerSC) }} + {{- fail (printf "nodeGroup %q: storageClass %q differs from golden image %q storageClass %q — across StorageClasses CDI either falls back to a host-assisted copy over the pod network (different CSI drivers or volume modes) or leaves a clone into the other class to the CSI driver.%s Neither side can be moved in place: this golden's class is fixed for its lifetime, because a DataVolume spec cannot be patched, and the pool's storageClass is immutable on an existing pool. So add a NEW kubernetes-worker-image catalog entry with a different (schematicID, version) on storageClass %q (kubernetesWorkerImage.images in the cozystack-platform Package values) and point this pool's osImage.builtin at it; or, once no pool clones this golden, delete it and let the catalog recreate it on that class. Switching this pool to osImage.factory also resolves it." .groupName $workerSC $goldenName $goldenSC $defaultNote $workerSC) }} {{- end }} {{- $goldenStorage := dig "spec" "storage" "resources" "requests" "storage" "" $goldenDV }} {{- /* Same authority argument as the class one line up: a golden whose spec @@ -545,10 +550,10 @@ spec: A builtin pool instead renders the class the guard above resolved for it. Omitting the field there would let the disk bind on whatever the - cluster default is at bind time, and CDI takes the storage-layer path - only when source and target share a class -- so a default that moved - after the pool's earlier disks were bound turns the next clone into a - host-assisted copy over the pod network, the transfer the guard exists + cluster default is at bind time -- so a default that moved after the + pool's earlier disks were bound turns the next clone into a + cross-class one, which CDI either runs as a host-assisted copy over + the pod network or leaves to the CSI driver, the clone the guard exists to refuse, while the guard reports no drift because it compared the class those earlier disks sit on. With the class empty and no cluster to resolve against the field is still omitted, so the HTTP path does diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_default_storageclass_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_default_storageclass_test.yaml index f016597fc..7bdddb41e 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_default_storageclass_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_default_storageclass_test.yaml @@ -101,10 +101,9 @@ kubernetesProvider: status: {phase: Succeeded} tests: - # Guaranteed NOT to be a clone, and the case an unresolved empty class waved - # through: the pool is on an explicit class, the golden is on the cluster - # default, and the two differ. CDI would fall back to the host-assisted copy - # over the pod network — silently, which is the transfer this feature removes. + # A cross-class clone, and the case an unresolved empty class waved through: + # the pool is on an explicit class, the golden is on the cluster default, and + # the two differ. - it: fails when the golden takes a cluster default class the pool does not use set: storageClass: local @@ -145,7 +144,7 @@ tests: # Reading the spec alone would call this golden "on the cluster default", which # is `replicated` here, and clone it for a `replicated` pool. Its PVC is on - # `local`, so that clone would silently be a host-assisted copy. + # `local`, so that clone would cross classes. - it: takes a blank-spec golden's class from its PVC, not from today's default set: storageClass: replicated diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_frozen_storageclass_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_frozen_storageclass_test.yaml index 4077d3dd4..84419168b 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_frozen_storageclass_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_frozen_storageclass_test.yaml @@ -189,7 +189,7 @@ tests: documentSelector: {path: kind, value: KubevirtMachineTemplate} # And the disk binds on the class the guard just compared. Leaving the field # out here passed the guard and then bound the next disk on `fast-nvme`, one - # class away from the golden, which is the host-assisted copy the guard is + # class away from the golden, which is the cross-class clone the guard is # there to refuse. - equal: path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName @@ -243,7 +243,7 @@ tests: version: v1.13.6 asserts: - failedTemplate: - errorPattern: 'An empty storageClass was resolved to the cluster default StorageClass "fast-nvme"' + errorPattern: 'An empty storageClass was resolved to "fast-nvme", the class CDI binds a disk that names none on\.' # With both sides explicit nothing was resolved from the default, and the # sentence is absent rather than reporting an empty lookup. The two sentences @@ -257,7 +257,7 @@ tests: version: v1.13.6 asserts: - failedTemplate: - errorPattern: 'host-assisted copy over the pod network\. Neither side can be moved in place' + errorPattern: 'to the CSI driver\. Neither side can be moved in place' # The size floor reads the PVC when the DataVolume spec omits a size, so a # golden it could size no longer slips past unmeasured. diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml index b076f50bd..41745ef6c 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml @@ -367,8 +367,8 @@ tests: value: talos-worker-norunning-v1.0.0 documentSelector: {path: kind, value: KubevirtMachineTemplate} - # Cross-StorageClass clone: CDI would silently fall back to a host-assisted copy - # over the pod network, the exact transfer osImage.builtin exists to avoid. + # Cross-StorageClass clone: a host-assisted copy over the pod network, or a + # clone left to the CSI driver to support, so the pool is refused. - it: osImage.builtin fails when the pool storageClass differs from the golden's set: storageClass: local diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml index d1b6f4652..d6d726786 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml @@ -107,7 +107,7 @@ tests: storageclass.kubevirt.io/is-default-virt-class: "true" asserts: - failedTemplate: - errorPattern: 'storageClass "virt-newer" differs from golden image .* storageClass "virt-default".* An empty storageClass was resolved to the cluster default StorageClass "virt-newer"' + errorPattern: 'storageClass "virt-newer" differs from golden image .* storageClass "virt-default".* An empty storageClass was resolved to "virt-newer"' - it: ignores a virtualization default annotation that is not "true" kubernetesProvider: diff --git a/packages/apps/kubernetes-nodes/values.schema.json b/packages/apps/kubernetes-nodes/values.schema.json index 0c24c9eb9..ea73ac2d7 100644 --- a/packages/apps/kubernetes-nodes/values.schema.json +++ b/packages/apps/kubernetes-nodes/values.schema.json @@ -14,7 +14,7 @@ ] }, "storageClass": { - "description": "StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.", + "description": "StorageClass for the worker node system disk. When empty, CDI picks the class: the one annotated `storageclass.kubevirt.io/is-default-virt-class`, else the cluster default. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: the chart refuses a pool whose class differs from its golden's, because across classes CDI either falls back to a host-assisted copy over the pod network (different CSI drivers or volume modes) or leaves a clone into the other class to the CSI driver. Leaving it empty makes the same comparison against the class CDI would pick. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.", "type": "string", "default": "replicated", "x-kubernetes-validations": [ diff --git a/packages/apps/kubernetes-nodes/values.yaml b/packages/apps/kubernetes-nodes/values.yaml index 94e1a50f9..e7fd74f94 100644 --- a/packages/apps/kubernetes-nodes/values.yaml +++ b/packages/apps/kubernetes-nodes/values.yaml @@ -22,7 +22,7 @@ cluster: "" ## @section Common Parameters ## -## @param {string} storageClass - StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. +## @param {string} storageClass - StorageClass for the worker node system disk. When empty, CDI picks the class: the one annotated `storageclass.kubevirt.io/is-default-virt-class`, else the cluster default. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: the chart refuses a pool whose class differs from its golden's, because across classes CDI either falls back to a host-assisted copy over the pod network (different CSI drivers or volume modes) or leaves a clone into the other class to the CSI driver. Leaving it empty makes the same comparison against the class CDI would pick. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. ## @x-cozystack-options {source: storageclass} ## @immutable storageClass: replicated diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 8ea4a054a..0d0776738 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -765,10 +765,10 @@ backupStrategyController: {} # cozystack.kubernetes-worker-image to bundles.enabledPackages -- the package is # opt-in and this key alone does not enable it. The two knobs that decide # whether the feature is usable at all are imageFactoryURL (an air-gapped or -# rate-limited cluster must repoint the single import) and storageClass (CDI -# cannot clone across StorageClasses, so a cluster whose worker pools are not on -# `replicated` must move the golden onto their class). Empty (the default) -# forwards nothing and keeps the child chart defaults. +# rate-limited cluster must repoint the single import) and storageClass (a +# worker pool refuses a golden on another StorageClass, so a cluster whose +# worker pools are not on `replicated` must move the golden onto their class). +# Empty (the default) forwards nothing and keeps the child chart defaults. kubernetesWorkerImage: {} # Pod scheduling configuration scheduling: diff --git a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml index ba1d7e01e..6525d3fb7 100644 --- a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml +++ b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml @@ -21,7 +21,7 @@ spec: singular: kubernetesnodes plural: kubernetesnodeses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","maximum":2147483647,"minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration for this pool.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"sysctls":{"description":"Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: \"11255\"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools.","type":"object","default":{},"additionalProperties":{"type":"string"}},"version":{"description":"Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} + {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, CDI picks the class: the one annotated `storageclass.kubevirt.io/is-default-virt-class`, else the cluster default. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: the chart refuses a pool whose class differs from its golden's, because across classes CDI either falls back to a host-assisted copy over the pod network (different CSI drivers or volume modes) or leaves a clone into the other class to the CSI driver. Leaving it empty makes the same comparison against the class CDI would pick. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","maximum":2147483647,"minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration for this pool.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"sysctls":{"description":"Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: \"11255\"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools.","type":"object","default":{},"additionalProperties":{"type":"string"}},"version":{"description":"Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} release: labels: sharding.fluxcd.io/key: tenants diff --git a/packages/system/kubernetes-worker-image/values.yaml b/packages/system/kubernetes-worker-image/values.yaml index 169632ba6..58e1b3646 100644 --- a/packages/system/kubernetes-worker-image/values.yaml +++ b/packages/system/kubernetes-worker-image/values.yaml @@ -90,7 +90,7 @@ ## @param {string} imageFactoryURL - Default base URL of the Talos Image Factory serving the raw worker OS image (`/image///openstack-amd64.raw.xz`). Overridable per image. No trailing slash. imageFactoryURL: "https://factory.talos.dev" -## @param {string} storageClass - Default StorageClass for the golden image(s). Must match the StorageClass of the worker disks that clone it: CDI cannot CSI-clone across StorageClasses and would silently fall back to a host-assisted copy over the pod network, so the tenant render rejects a mismatch outright. Overridable per image. +## @param {string} storageClass - Default StorageClass for the golden image(s). Must match the StorageClass of the worker disks that clone it: across classes CDI either falls back to a host-assisted copy over the pod network or leaves the clone to the CSI driver, so the tenant render rejects a mismatch outright. Overridable per image. storageClass: "replicated" ## @param {quantity} storage - Default storage size allocated for each golden image. Must be >= the imported raw image virtual size and <= the smallest worker diskSize that clones it. The Talos v1.13.x openstack raw is ~4.15 GiB, so 6Gi leaves headroom. This size is also the unit of the transient cost every clone pays: CDI populates a worker disk larger than the golden by cloning at the golden's size and then growing it, so one worker in flight occupies the golden plus a temporary clone PVC plus its own diskSize, times the replica count of a replicated StorageClass. Keeping this at the raw image's size rather than rounding it up is therefore worth doing -- it is paid once per golden and again per concurrent clone. Overridable per image, but only when the entry is added: a DataVolume spec cannot be patched in place, so raising this on an entry whose golden already exists is rejected on the next upgrade — a Talos version whose raw outgrows it needs a new `images[]` entry, which it gets anyway since the name is keyed on the version. diff --git a/packages/system/vm-default-images/values.yaml b/packages/system/vm-default-images/values.yaml index a506ad1c0..9203bcfd9 100644 --- a/packages/system/vm-default-images/values.yaml +++ b/packages/system/vm-default-images/values.yaml @@ -9,7 +9,7 @@ ## images that did import stay usable meanwhile. ## -## @param {string} storageClass - Default StorageClass for all images. If empty, uses the cluster default StorageClass. +## @param {string} storageClass - Default StorageClass for all images. If empty, CDI picks the class: the one annotated `storageclass.kubevirt.io/is-default-virt-class`, else the cluster default. ## NOTE: The default set of images requires approximately 320Gi of storage (16 images × 20Gi each). ## Adjust the image list or per-image storage sizes to match your cluster capacity. storageClass: "replicated" From 76d68a919f023a5e92436569bd9d65f4c1b677fb Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:16:47 +0300 Subject: [PATCH 6/6] test(e2e): pin the importer restarts an unreachable VMDisk reports The kubernetes-nodes guard and the image option list call an import failing once its importer has restarted three times and is not running, and the vm-disk guard once it has restarted three times, because CDI keeps an unreachable URL in progress instead of moving it to Failed. Assert that CDI reports both on the DataVolume of the unreachable VMDisk, so a CDI change that stops doing so turns this suite red rather than leaving the guards blind. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../vminstance/chainsaw-test.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml b/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml index 691d45302..a24a0073a 100644 --- a/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml @@ -91,6 +91,30 @@ spec: (conditions[?type == 'WorkloadsReady']): - status: "False" reason: DataVolumeNotReady + # The golden image guard in the kubernetes-nodes chart and the image option + # list call an import failing once its importer has restarted three times and + # is not running, and the vm-disk guard once it has restarted three times, + # because CDI keeps an unreachable URL in progress instead of moving it to + # Failed. This pins that CDI does report both. The default kubelet back-off + # before the third restart adds up to 70s (10s, 20s, 40s), and the timeout + # leaves room for the importer's own runs on top of that. The count is + # compared as a string, because Chainsaw v0.2.15 mis-evaluates numeric + # literals; 'null' in the list makes an absent count, which CDI omits at + # zero, fail too. + - name: wait-importer-restarts + try: + - assert: + timeout: 4m + resource: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-unreachable + status: + phase: ImportInProgress + (contains(['null', '0', '1', '2'], to_string(restartCount))): false + (conditions[?type == 'Running']): + - status: "False" --- # VMDisk clone paths: source.image clones a golden image from cozy-public, and # source.disk clones that disk in turn, so only the golden image downloads.