From 8dcf950842cd6431d3c24adf075252a2b0c98333 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:12:41 +0300 Subject: [PATCH] fix(kubernetes-nodes)!: resolve an empty class to CDI's virtualization default CDI binds a kubevirt-content DataVolume that names no class on the class annotated storageclass.kubevirt.io/is-default-virt-class before it falls back to the Kubernetes default. The worker image guard resolved an empty class to the Kubernetes default only, so on a cluster carrying both it compared the golden against a class the golden was not on and refused a pool that matched it. CDI's fallback reads only the GA default annotation, so the beta one no longer ties with it: a class carrying only the beta annotation is picked only when no class carries the GA one. CDI then creates no claim at all for a DataVolume naming neither a class nor access modes, so such a pool gets that class only because the chart writes it onto the worker disk. The resolution also refuses a pool that rendered before: one with an empty class that has not cloned a disk yet, whose golden is on the Kubernetes default, on a cluster where another class carries the virtualization default. Its storageClass is immutable, so it has to point osImage.builtin at a catalog entry on the virtualization default or switch to osImage.factory. A pool that already cloned reads its class back from its own disk's claim and keeps rendering. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../kubernetes-nodes/templates/_helpers.tpl | 44 +++-- .../kubernetes-nodes/templates/nodegroup.yaml | 3 +- .../worker_image_beta_default_class_test.yaml | 81 ++++++++ ...worker_image_two_default_classes_test.yaml | 19 ++ .../worker_image_virt_default_class_test.yaml | 183 ++++++++++++++++++ 5 files changed, 313 insertions(+), 17 deletions(-) create mode 100644 packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml create mode 100644 packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml diff --git a/packages/apps/kubernetes-nodes/templates/_helpers.tpl b/packages/apps/kubernetes-nodes/templates/_helpers.tpl index 5e317f8e6..262b947c8 100644 --- a/packages/apps/kubernetes-nodes/templates/_helpers.tpl +++ b/packages/apps/kubernetes-nodes/templates/_helpers.tpl @@ -441,18 +441,25 @@ still fails. {{- end }} {{- /* -Name of the cluster's default StorageClass, or the empty string when there is -none (and always under `helm template`, which has no cluster to read). +Name of the StorageClass CDI binds a kubevirt-content DataVolume on when it +names no class, or the empty string when there is none (and always under `helm +template`, which has no cluster to read). CDI takes a class annotated +storageclass.kubevirt.io/is-default-virt-class first and only then the +Kubernetes default (GetStorageClassByNameWithVirtFallback in CDI's +pkg/controller/common/util.go), and both the goldens and the worker disks are +kubevirt content. An empty storageClass is a documented, schema-valid setting on both a worker pool and a worker image catalog entry, and it means "the cluster default". Without resolving it the golden-versus-pool StorageClass comparison simply skips whenever -either side is empty, which is the one corner where skipping is worst: CDI then -falls back to a host-assisted copy over the pod network, silently, and that copy -is the transfer the clone path exists to remove. +either side is empty, and lets a cross-class clone through unchecked. -Both the current annotation and its beta predecessor count, because clusters -provisioned years apart carry different ones and Kubernetes still honours both. +The beta predecessor of the Kubernetes annotation is read last, only when no +class carries the current one. CDI does not read it: with neither of the other +two defaults it creates no claim at all for a DataVolume that names no class +and no access modes (pvcFromStorage, ErrStorageClassNotFound). Read here, it +still gives a pool on such a cluster a class, written onto the worker disk +explicitly. More than one class may carry the annotation at once. Kubernetes permits that -- it is the normal state midway through swapping a cluster's default -- and @@ -463,16 +470,21 @@ that is actually in force. Timestamps are RFC3339 in UTC, so comparing them as strings is comparing them chronologically. */ -}} {{- define "kubernetes-nodes.defaultStorageClassName" -}} -{{- $classes := lookup "storage.k8s.io/v1" "StorageClass" "" "" -}} +{{- $classes := dig "items" (list) ((lookup "storage.k8s.io/v1" "StorageClass" "" "") | default dict) -}} {{- $name := "" -}} -{{- $createdAt := "" -}} -{{- range (dig "items" (list) ($classes | default dict)) -}} -{{- $annotations := dig "metadata" "annotations" (dict) . -}} -{{- if or (eq (dig "storageclass.kubernetes.io/is-default-class" "" $annotations | toString) "true") (eq (dig "storageclass.beta.kubernetes.io/is-default-class" "" $annotations | toString) "true") -}} -{{- $at := dig "metadata" "creationTimestamp" "" . | toString -}} -{{- if or (not $name) (gt $at $createdAt) -}} -{{- $name = dig "metadata" "name" "" . | toString -}} -{{- $createdAt = $at -}} +{{- range list "storageclass.kubevirt.io/is-default-virt-class" "storageclass.kubernetes.io/is-default-class" "storageclass.beta.kubernetes.io/is-default-class" -}} +{{- $key := . -}} +{{- $createdAt := "" -}} +{{- if not $name -}} +{{- range $classes -}} +{{- $annotations := dig "metadata" "annotations" (dict) . -}} +{{- if eq (dig $key "" $annotations | toString) "true" -}} +{{- $at := dig "metadata" "creationTimestamp" "" . | toString -}} +{{- if or (not $name) (gt $at $createdAt) -}} +{{- $name = dig "metadata" "name" "" . | toString -}} +{{- $createdAt = $at -}} +{{- end -}} +{{- end -}} {{- end -}} {{- end -}} {{- end -}} diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index bbf8c24d8..39d71b7be 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -540,7 +540,8 @@ spec: rejects RWX volumes that are not on a DRBD-backed StorageClass, which is why the pool's storageClass defaults to a replicated/DRBD class rather than to the cluster default. An empty storageClass - omits the field so the cluster default applies. + omits the field, and CDI binds the disk on its virtualization + default or, without one, the Kubernetes default. A builtin pool instead renders the class the guard above resolved for it. Omitting the field there would let the disk bind on whatever the diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml new file mode 100644 index 000000000..d1f7787e8 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/worker_image_beta_default_class_test.yaml @@ -0,0 +1,81 @@ +suite: osImage.builtin default-StorageClass selection with only the beta annotation + +# CDI does not read the beta annotation, so the pool's class comes from the +# chart alone, written onto the worker disk explicitly. + +templates: + - templates/nodegroup.yaml + +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test + +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: [ingress-nginx] + osImage: + builtin: {} + +kubernetesProvider: + scheme: + "instancetype.kubevirt.io/v1beta1/VirtualMachineClusterInstancetype": + gvr: {group: instancetype.kubevirt.io, version: v1beta1, resource: virtualmachineclusterinstancetypes} + namespaced: false + "cluster.x-k8s.io/v1beta1/MachineSet": + gvr: {group: cluster.x-k8s.io, version: v1beta1, resource: machinesets} + namespaced: true + "infrastructure.cluster.x-k8s.io/v1alpha1/KubevirtMachineTemplate": + gvr: {group: infrastructure.cluster.x-k8s.io, version: v1alpha1, resource: kubevirtmachinetemplates} + namespaced: true + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes} + namespaced: true + "storage.k8s.io/v1/StorageClass": + gvr: {group: storage.k8s.io, version: v1, resource: storageclasses} + namespaced: false + objects: + - kind: VirtualMachineClusterInstancetype + apiVersion: instancetype.kubevirt.io/v1beta1 + metadata: {name: u1.medium} + - kind: MachineSet + apiVersion: cluster.x-k8s.io/v1beta1 + metadata: {name: ignored-ms, namespace: tenant-test} + spec: {template: {spec: {infrastructureRef: {kind: Other, name: ignored}}}} + - kind: KubevirtMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + metadata: {name: ignored-kmt, namespace: tenant-test} + # The outgoing default, created first. + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: beta-only + creationTimestamp: "2026-01-01T00:00:00Z" + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515-v1.13.6 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: beta-only + status: {phase: Succeeded} + +tests: + - it: resolves an empty pool storageClass to a class with only the beta annotation + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: beta-only + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml index 1c30a2d2c..5613c6302 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_two_default_classes_test.yaml @@ -114,3 +114,22 @@ tests: asserts: - failedTemplate: errorPattern: 'requested golden image "talos-worker-elsewhere-v1\.0\.0" not found' + + # CDI reads only the GA annotation when it picks a class itself, so a newer + # class carrying only the beta one does not displace a GA default. + - it: prefers a GA default over a newer class with only the beta annotation + kubernetesProvider: + objects: + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: beta-only + creationTimestamp: "2026-09-01T00:00:00Z" + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: new + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml new file mode 100644 index 000000000..d1b6f4652 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/worker_image_virt_default_class_test.yaml @@ -0,0 +1,183 @@ +suite: osImage.builtin default-StorageClass selection with a virtualization default + +# CDI resolves a DataVolume with no class and contentType kubevirt to the class +# annotated storageclass.kubevirt.io/is-default-virt-class before it looks at the +# Kubernetes default, so a cluster that carries both binds the golden and the +# worker disks on the virtualization default. The clone guard has to resolve an +# empty class the same way, or it compares the golden against a class it is not +# on. + +templates: + - templates/nodegroup.yaml + +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test + +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: [ingress-nginx] + osImage: + builtin: {} + +kubernetesProvider: + scheme: + "instancetype.kubevirt.io/v1beta1/VirtualMachineClusterInstancetype": + gvr: {group: instancetype.kubevirt.io, version: v1beta1, resource: virtualmachineclusterinstancetypes} + namespaced: false + "cluster.x-k8s.io/v1beta1/MachineSet": + gvr: {group: cluster.x-k8s.io, version: v1beta1, resource: machinesets} + namespaced: true + "infrastructure.cluster.x-k8s.io/v1alpha1/KubevirtMachineTemplate": + gvr: {group: infrastructure.cluster.x-k8s.io, version: v1alpha1, resource: kubevirtmachinetemplates} + namespaced: true + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes} + namespaced: true + "storage.k8s.io/v1/StorageClass": + gvr: {group: storage.k8s.io, version: v1, resource: storageclasses} + namespaced: false + "v1/PersistentVolumeClaim": + gvr: {group: "", version: v1, resource: persistentvolumeclaims} + namespaced: true + objects: + - kind: VirtualMachineClusterInstancetype + apiVersion: instancetype.kubevirt.io/v1beta1 + metadata: {name: u1.medium} + - kind: MachineSet + apiVersion: cluster.x-k8s.io/v1beta1 + metadata: {name: ignored-ms, namespace: tenant-test} + spec: {template: {spec: {infrastructureRef: {kind: Other, name: ignored}}}} + - kind: KubevirtMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + metadata: {name: ignored-kmt, namespace: tenant-test} + # The Kubernetes default, created after the virtualization default so that + # creation order alone would pick it. + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: kube-default + creationTimestamp: "2026-06-01T00:00:00Z" + annotations: + storageclass.kubernetes.io/is-default-class: "true" + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: virt-default + creationTimestamp: "2026-01-01T00:00:00Z" + annotations: + storageclass.kubevirt.io/is-default-virt-class: "true" + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515-v1.13.6 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: virt-default + status: {phase: Succeeded} + +tests: + - it: resolves an empty pool storageClass to the virtualization default + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: virt-default + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: takes the most recently created of two virtualization defaults + kubernetesProvider: + objects: + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: virt-newer + creationTimestamp: "2026-03-01T00:00:00Z" + annotations: + storageclass.kubevirt.io/is-default-virt-class: "true" + asserts: + - failedTemplate: + errorPattern: 'storageClass "virt-newer" differs from golden image .* storageClass "virt-default".* An empty storageClass was resolved to the cluster default StorageClass "virt-newer"' + + - it: ignores a virtualization default annotation that is not "true" + kubernetesProvider: + objects: + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: virt-off + creationTimestamp: "2026-05-01T00:00:00Z" + annotations: + storageclass.kubevirt.io/is-default-virt-class: "false" + asserts: + - notFailedTemplate: {} + + # Before the virtualization default was read, a pool that has not cloned a + # disk yet resolved to the Kubernetes default here and rendered. + - it: refuses a pool whose golden is on the Kubernetes default instead + set: + osImage: + builtin: {schematicID: kube, version: v1.0.0} + kubernetesProvider: + objects: + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-kube-v1.0.0 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: kube-default + status: {phase: Succeeded} + asserts: + - failedTemplate: + errorPattern: 'storageClass "virt-default" differs from golden image .* storageClass "kube-default".* An empty storageClass was resolved to "virt-default"' + + # A pool that already cloned reads its class back from its own disk's claim + # and never reaches the default lookup. + - it: keeps rendering a pool that already cloned from a golden on the Kubernetes default + set: + osImage: + builtin: {schematicID: kube, version: v1.0.0} + kubernetesProvider: + objects: + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: talos-worker-kube-v1.0.0 + namespace: cozy-public + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: kube-default + status: {phase: Succeeded} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: disk-system-kubernetes-myk8s-md0-abcde + namespace: tenant-test + annotations: + kubernetes-worker-image.cozystack.io/golden: talos-worker-kube-v1.0.0 + kubernetes-worker-image.cozystack.io/pool: kubernetes-myk8s-md0 + status: {phase: Succeeded} + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: disk-system-kubernetes-myk8s-md0-abcde, namespace: tenant-test} + spec: {storageClassName: kube-default} + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName + value: kube-default + documentSelector: {path: kind, value: KubevirtMachineTemplate}