From 513d250a68b1bbffd16f0504994ca245690537c7 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 7 Oct 2026 14:10:08 +0300 Subject: [PATCH] fix(kubernetes-nodes): refuse a golden whose importer keeps failing The worker image guard refused only a golden in phase Failed, which CDI reaches on a fatal error such as a checksum mismatch. An unreachable URL restarts the importer with back-off while the DataVolume stays in progress, so a pool pinned to such a golden waited forever for its first disk. Refuse an importer that restarted at least three times and is not running, under the same already-cloned exemption as Failed. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../kubernetes-nodes/templates/nodegroup.yaml | 19 +++- .../tests/worker_image_cloned_pool_test.yaml | 36 ++++++- .../tests/worker_image_source_test.yaml | 96 +++++++++++++++++++ .../kubernetes-worker-image/values.yaml | 8 +- 4 files changed, 151 insertions(+), 8 deletions(-) diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 440bbfa05..bbf8c24d8 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -302,7 +302,7 @@ spec: and no CR edit is needed to trigger the abort, only the next periodic reconcile. The comment below rejects exactly that blast radius for a mid-import golden; applying the same reasoning to a golden that is - absent or Failed is what it costs to be consistent. + absent or failing to import is what it costs to be consistent. So the loud failure stays where it is the only signal — a pool still waiting for its first disk — and a pool that is past that point is @@ -367,21 +367,32 @@ spec: {{- fail (printf "nodeGroup %q: osImage.builtin requested golden image %q not found in cozy-public, neither as a DataVolume nor as a PVC — enable the cozystack.kubernetes-worker-image catalog via bundles.enabledPackages with a matching (schematicID, version) entry, or switch this pool to osImage.factory." .groupName $goldenName) }} {{- end }} {{- else }} - {{- /* Fail only on the golden's terminal failure phase, not on "not yet + {{- /* Fail only on a golden whose import is failing, not on "not yet Succeeded". A golden mid-import is transient: CDI holds the worker DataVolume until the source is populated, which is the correct behaviour and recovers unattended. Hard-failing the render there would abort the ENTIRE KubernetesNodes HelmRelease — every resource, not just this disk — for every pool pinned to that golden, blocking unrelated - scale operations for the length of an import. Only Failed, which - never recovers without operator action, justifies that blast radius. + scale operations for the length of an import. Failed never recovers + without operator action, but CDI reaches it only on a fatal error such + as a checksum mismatch: an unreachable URL restarts the importer with + back-off while the phase stays in progress. So an importer that + restarted at least three times and is not running now is refused too; + that refusal lifts on its own once an attempt succeeds. Falling back to the HTTP import is deliberately not an option either: it would make disk-system's source depend on cluster state, so the source — and with it the content-hashed KubevirtMachineTemplate name — would flip once the golden went Ready and roll every worker in the pool. */}} {{- $goldenPhase := dig "status" "phase" "" $goldenDV | toString }} + {{- $goldenRestarts := dig "status" "restartCount" 0 $goldenDV | int }} + {{- $goldenRunning := "" }} + {{- range (dig "status" "conditions" (list) $goldenDV) }} + {{- if eq (dig "type" "" . | toString) "Running" }}{{- $goldenRunning = dig "status" "" . | toString }}{{- end }} + {{- end }} {{- if and (eq $goldenPhase "Failed") (not $poolCloned) }} {{- fail (printf "nodeGroup %q: golden DataVolume %q in cozy-public failed to import (phase Failed) — cloning it would block this pool's workers indefinitely. Check the importer pod in cozy-public, then delete the DataVolume to retry the import." .groupName $goldenName) }} + {{- else if and (ne $goldenPhase "Succeeded") (ge $goldenRestarts 3) (eq $goldenRunning "False") (not $poolCloned) }} + {{- fail (printf "nodeGroup %q: golden DataVolume %q in cozy-public keeps failing to import (%d importer restarts) — cloning it would block this pool's workers while it does. The refusal lifts by itself while a new import attempt is running, and for good once one succeeds. The importer is retried with a growing back-off, and a recreated importer pod counts as not running until its container starts, so after the source is fixed the refusal lasts at least until the current back-off delay ends and the next importer starts. Check the importer pod logs in cozy-public." .groupName $goldenName $goldenRestarts) }} {{- end }} {{- /* CDI takes the storage-layer CSI-clone path only when source and target share a StorageClass; across classes it silently falls back to a diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml index 1bc4adb8a..8467ae6df 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_cloned_pool_test.yaml @@ -1,6 +1,6 @@ suite: osImage.builtin golden guards on a pool that has already cloned -# The absence and Failed guards abort the ENTIRE KubernetesNodes HelmRelease, and +# The absence and failed-import guards abort the ENTIRE KubernetesNodes HelmRelease, and # for a pool whose workers all cloned long ago that price buys nothing: replica # changes, kubelet retuning and MachineHealthCheck timeouts have no stake in the # golden, and no CR edit is needed to trigger the abort — the next periodic @@ -152,6 +152,30 @@ kubernetesProvider: resources: {requests: {storage: 6Gi}} storageClassName: replicated status: {phase: Failed} + # md0's own disk against a golden whose re-import is now stuck: the pool + # already holds its copy, so it needs nothing from the source. + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: disk-system-kubernetes-myk8s-md0-pqrst + namespace: tenant-test + annotations: + kubernetes-worker-image.cozystack.io/golden: talos-worker-stucklater-v1.0.0 + kubernetes-worker-image.cozystack.io/pool: kubernetes-myk8s-md0 + status: {phase: Succeeded} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-stucklater-v1.0.0, namespace: cozy-public} + spec: + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: replicated + status: + phase: ImportInProgress + # A string: helm-unittest's lookup cannot deep-copy an int, and | int takes either. + restartCount: "7" + conditions: + - {type: Running, status: "False", reason: Error} tests: - it: renders when the golden is gone but this pool already cloned it @@ -217,3 +241,13 @@ tests: asserts: - failedTemplate: errorPattern: 'golden DataVolume "talos-worker-sibling-v1\.0\.0" in cozy-public failed to import' + + - it: renders when the golden importer keeps failing but this pool already cloned it + set: + osImage: + builtin: {schematicID: stucklater, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-stucklater-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml b/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml index 2794e824b..b076f50bd 100644 --- a/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/worker_image_source_test.yaml @@ -119,6 +119,54 @@ kubernetesProvider: storageClassName: replicated status: phase: Failed + # Goldens whose importer has restarted. CDI leaves an unreachable URL in + # progress and restarts the importer instead of reaching Failed, so restarts + # plus an importer that is not running now is what marks one as failing. + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-stuck-v1.0.0, namespace: cozy-public} + spec: &goldenSpec + storage: + resources: {requests: {storage: 6Gi}} + storageClassName: replicated + status: + phase: ImportInProgress + # A string: helm-unittest's lookup cannot deep-copy an int, and | int takes either. + restartCount: "3" + conditions: + - {type: Running, status: "False", reason: Error} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-recovered-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: + phase: ImportInProgress + restartCount: "5" + conditions: + - {type: Running, status: "True", reason: Pod is running} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-flaky-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: + phase: ImportInProgress + restartCount: "2" + conditions: + - {type: Running, status: "False", reason: Error} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-done-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: + phase: Succeeded + restartCount: "4" + conditions: + - {type: Running, status: "False", reason: Completed} + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: talos-worker-norunning-v1.0.0, namespace: cozy-public} + spec: *goldenSpec + status: {phase: ImportInProgress, restartCount: "3"} # A populated golden that declares no storage request at all. The size guard # reads spec.storage.resources.requests.storage, so its presence gate is the # branch that decides between comparing and skipping; without a fixture the @@ -271,6 +319,54 @@ tests: - failedTemplate: errorPattern: 'golden DataVolume "talos-worker-failed-v1\.0\.0" in cozy-public failed to import \(phase Failed\)' + - it: osImage.builtin fails the render when the golden importer keeps failing + set: + osImage: + builtin: {schematicID: stuck, version: v1.0.0} + asserts: + - failedTemplate: + errorPattern: 'golden DataVolume "talos-worker-stuck-v1\.0\.0" in cozy-public keeps failing to import \(3 importer restarts\).*lifts by itself.*a recreated importer pod counts as not running until its container starts' + + - it: osImage.builtin clones a golden whose importer restarted but is running again + set: + osImage: + builtin: {schematicID: recovered, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-recovered-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: osImage.builtin clones a golden whose importer failed fewer than three times + set: + osImage: + builtin: {schematicID: flaky, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-flaky-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: osImage.builtin clones a golden that Succeeded after its importer restarted + set: + osImage: + builtin: {schematicID: done, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-done-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + + - it: osImage.builtin clones a golden whose importer restarted and reports no Running condition + set: + osImage: + builtin: {schematicID: norunning, version: v1.0.0} + asserts: + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.pvc.name + value: talos-worker-norunning-v1.0.0 + documentSelector: {path: kind, value: KubevirtMachineTemplate} + # Cross-StorageClass clone: CDI would silently fall back to a host-assisted copy # over the pod network, the exact transfer osImage.builtin exists to avoid. - it: osImage.builtin fails when the pool storageClass differs from the golden's diff --git a/packages/system/kubernetes-worker-image/values.yaml b/packages/system/kubernetes-worker-image/values.yaml index 67fb268ba..169632ba6 100644 --- a/packages/system/kubernetes-worker-image/values.yaml +++ b/packages/system/kubernetes-worker-image/values.yaml @@ -30,10 +30,12 @@ ## `talos-worker--` from cozy-public. The tenant render fails ## rather than degrade silently when the referenced (schematicID, version) has no ## matching entry in this catalog, when its golden terminally failed to import -## (`phase: Failed`), or when the node group's storageClass differs from the +## (`phase: Failed`) or keeps failing to (its importer restarted at least 3 times +## and is not running), or when the node group's storageClass differs from the ## golden's — so keep the catalog in sync with the (schematicID, version) pairs and -## StorageClasses that tenant node groups reference. A golden that is merely -## mid-import is not an error: CDI holds the worker disk until the source populates. +## StorageClasses that tenant node groups reference. A golden still importing, with +## its importer running or restarted fewer than 3 times, is not an error: CDI holds +## the worker disk until the source populates. ## Goldens carry `helm.sh/resource-policy: keep`, so removing an `images[]` entry ## stops managing that golden rather than deleting it. Deleting one by hand while a ## worker pool still clones it fails that pool's render on its next values or chart