mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(kubernetes): detach volumes when the infra PVC is missing (#5249)
## What this PR does routes an infra PVC `NotFound` in `ControllerUnpublishVolume` through the existing upstream hotplug detach path. a quota-rejected PVC can still have a persistent entry in the VM spec, so returning success before detach leaves a stale volume after the DataVolume is deleted. adds regression coverage for a pending hotplug volume, an already detached volume, and a VMI that outlives its VM. existing NFS unpublish tests still pass. no dependency update is needed. Fixes https://github.com/cozystack/cozystack/issues/5248 validation in the kubevirt-csi-driver module: `go test -count=1 ./...` and `go vet ./...`. the new regression fails without the production change. root `make generate` and `make manifests` also pass without generated-file changes. ### Screenshots no UI changes. ### Downstream repositories - [x] No downstream repository is affected by this change - [ ] [cozystack/website](https://github.com/cozystack/website) - follow-up: - [ ] [cozystack/terraform-provider-cozystack](https://github.com/cozystack/terraform-provider-cozystack) - follow-up: - [ ] [cozystack/ansible-cozystack](https://github.com/cozystack/ansible-cozystack) - follow-up: - [ ] [cozystack/ccp](https://github.com/cozystack/ccp) - follow-up: - [ ] [cozystack/talm](https://github.com/cozystack/talm) - follow-up: - [ ] [cozystack/cozyhr](https://github.com/cozystack/cozyhr) - follow-up: - [ ] [cozystack/cozy-proxy](https://github.com/cozystack/cozy-proxy) - follow-up: - [ ] [cozystack/cozystack-telemetry-server](https://github.com/cozystack/cozystack-telemetry-server) - follow-up: - [ ] [cozystack/external-apps-example](https://github.com/cozystack/external-apps-example) - follow-up: - [ ] [cozystack/examples](https://github.com/cozystack/examples) - follow-up: - [ ] [cozystack/community](https://github.com/cozystack/community) - follow-up: ### Release note ```release-note fix(kubernetes): remove stale hotplug volumes when the infra PVC is missing, preventing deleted volumes from blocking subsequent attachments ``` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> <!-- coderabbit-ai-generated:v1 --> ## Summary by CodeRabbit * **Bug Fixes** * Volume unpublish requests now proceed through standard cleanup when the associated PVC is missing, rather than returning success immediately. This allows remaining teardown steps to run for detached volumes and VM-level resources. Other PVC lookup errors continue to be returned directly, without entering the missing-PVC cleanup path. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
2 files changed
+93
-12
No files matched your search
@@ -424,7 +424,8 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context
|
||||
pvc, err := w.infraClient.CoreV1().PersistentVolumeClaims(w.infraNamespace).Get(ctx, dvName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return &csi.ControllerUnpublishVolumeResponse{}, nil
|
||||
// A failed PVC creation can still leave a hotplug entry in the VM.
|
||||
return w.ControllerService.ControllerUnpublishVolume(ctx, req)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -310,15 +310,19 @@ const (
|
||||
testVolumeName = "pvc-0"
|
||||
)
|
||||
|
||||
// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its
|
||||
// VMI still reports the hot-plugged volume. The VMI drops the volume once
|
||||
// RemoveVolumeFromVMI is called, or when the garbage collector deletes a
|
||||
// VMI whose controlling VM is gone.
|
||||
// teardownClient keeps hotplug state until removal or garbage collection,
|
||||
// including when the VMI outlives its VM during CAPI/CAPK teardown.
|
||||
type teardownClient struct {
|
||||
kubevirtclient.Client
|
||||
vmiOwners []metav1.OwnerReference
|
||||
vmiRemovals int
|
||||
lookups int
|
||||
vm *kubevirtv1.VirtualMachine
|
||||
vmiOwners []metav1.OwnerReference
|
||||
detached bool
|
||||
vmiGone bool
|
||||
vmiGoneAfterCheck bool
|
||||
vmiLookups int
|
||||
vmRemovals int
|
||||
vmiRemovals int
|
||||
lookups int
|
||||
}
|
||||
|
||||
func (c *teardownClient) vmiGarbageCollected() bool {
|
||||
@@ -328,22 +332,46 @@ func (c *teardownClient) vmiGarbageCollected() bool {
|
||||
|
||||
func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) {
|
||||
c.lookups++
|
||||
if c.vm != nil {
|
||||
return c.vm, nil
|
||||
}
|
||||
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName)
|
||||
}
|
||||
|
||||
func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) {
|
||||
c.lookups++
|
||||
c.vmiLookups++
|
||||
if c.vmiGone {
|
||||
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, testVMName)
|
||||
}
|
||||
vmi := &kubevirtv1.VirtualMachineInstance{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners},
|
||||
}
|
||||
if c.vmiRemovals == 0 {
|
||||
vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
|
||||
if !c.detached && c.vmRemovals == 0 && c.vmiRemovals == 0 {
|
||||
vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, Phase: kubevirtv1.VolumePending, Reason: "FailedPvcNotFound", HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
|
||||
}
|
||||
return vmi, nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string, string) (bool, error) {
|
||||
return c.vm == nil || len(c.vm.Spec.Template.Spec.Volumes) == 0, nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) {
|
||||
return c.vmiRemovals > 0, nil
|
||||
removed := c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0
|
||||
if c.vmiGoneAfterCheck {
|
||||
c.vmiGone = true
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error {
|
||||
if namespace != testInfraNamespace || name != testVMName || opts.Name != testVolumeName {
|
||||
return errors.New("unexpected hotplug removal target")
|
||||
}
|
||||
c.vmRemovals++
|
||||
c.vm.Spec.Template.Spec.Volumes = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error {
|
||||
@@ -352,7 +380,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k
|
||||
}
|
||||
|
||||
func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error {
|
||||
if c.vmiRemovals > 0 || c.vmiGarbageCollected() {
|
||||
if c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() {
|
||||
return nil
|
||||
}
|
||||
return errors.New("timed out waiting for the VMI to drop the volume")
|
||||
@@ -410,6 +438,58 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerUnpublishVolumeMissingPVC(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vmGone bool
|
||||
detached bool
|
||||
vmiGone bool
|
||||
vmiGoneAfterCheck bool
|
||||
wantVMILookups int
|
||||
wantVMCalls int
|
||||
wantVMICalls int
|
||||
}{
|
||||
{name: "pending hotplug volume", wantVMCalls: 1},
|
||||
{name: "already detached", detached: true},
|
||||
{name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1, wantVMILookups: 1},
|
||||
{name: "VM gone, no attached volume", vmGone: true, detached: true},
|
||||
{name: "VM and VMI gone", vmGone: true, vmiGone: true},
|
||||
{name: "VMI disappears after attachment check", vmGone: true, vmiGoneAfterCheck: true, wantVMILookups: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := &teardownClient{detached: tt.detached, vmiGone: tt.vmiGone, vmiGoneAfterCheck: tt.vmiGoneAfterCheck}
|
||||
if !tt.vmGone {
|
||||
client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}}
|
||||
if !tt.detached {
|
||||
client.vm.Spec.Template.Spec.Volumes = []kubevirtv1.Volume{{
|
||||
Name: testVolumeName,
|
||||
VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: testVolumeName, Hotpluggable: true}},
|
||||
}}
|
||||
}
|
||||
}
|
||||
w := &WrappedControllerService{
|
||||
ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true),
|
||||
infraClient: fake.NewClientset(),
|
||||
infraNamespace: testInfraNamespace,
|
||||
}
|
||||
req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName}
|
||||
if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil {
|
||||
t.Fatalf("ControllerUnpublishVolume() error = %v", err)
|
||||
}
|
||||
if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls {
|
||||
t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls)
|
||||
}
|
||||
if client.vmiLookups != tt.wantVMILookups {
|
||||
t.Errorf("GetVirtualMachine calls = %d, want %d", client.vmiLookups, tt.wantVMILookups)
|
||||
}
|
||||
if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 {
|
||||
t.Error("hotplug volume remains in VM spec")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func nfsCNP(owners ...string) *unstructured.Unstructured {
|
||||
refs := make([]interface{}, 0, len(owners))
|
||||
for _, owner := range owners {
|
||||
|
||||
Reference in new issue
Block a user