diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go index acfbe5f22..9029c9ff3 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go @@ -411,65 +411,10 @@ func (w *WrappedControllerService) ControllerPublishVolume(ctx context.Context, }, nil } -// unpublishHotplugVolume delegates to upstream for the hot-unplug, then re-verifies -// that the volume is gone from VMI.Status.VolumeStatus before returning success. -// Upstream treats a missing parent VM as "detach succeeded", but during a CAPI/CAPK -// teardown the VMI (and its hot-plug pod) can outlive the VM: the hot-plug pod then -// keeps the infra storage device exclusively attached to the source host, and every -// subsequent attach of the same volume fails (on DRBD with "failed to set source -// device readwrite"). When the VMI still reports the hot-plug after upstream -// returned success, detach it from the VMI directly. Backport of -// kubevirt/csi-driver#184; drop once it is merged and the module is bumped past it. -func (w *WrappedControllerService) unpublishHotplugVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - if _, err := w.ControllerService.ControllerUnpublishVolume(ctx, req); err != nil { - return nil, err - } - - dvName := req.GetVolumeId() - _, vmName, err := cache.SplitMetaNamespaceKey(req.GetNodeId()) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to parse node ID %q: %v", req.GetNodeId(), err) - } - - // GetVirtualMachine on this client returns a VirtualMachineInstance, not a VM. - vmi, err := w.virtClient.GetVirtualMachine(ctx, w.infraNamespace, vmName) - if err != nil { - if errors.IsNotFound(err) { - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - return nil, status.Errorf(codes.Unavailable, - "cannot verify VMI %s/%s after unpublish of %s: %v", w.infraNamespace, vmName, dvName, err) - } - - stillAttached := false - for _, vs := range vmi.Status.VolumeStatus { - if vs.HotplugVolume != nil && vs.Name == dvName { - stillAttached = true - break - } - } - if !stillAttached { - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - - klog.Infof("Volume %s still reported by VMI %s/%s after unpublish – detaching from the VMI", dvName, w.infraNamespace, vmName) - if err := w.virtClient.RemoveVolumeFromVMI(ctx, w.infraNamespace, vmName, &kubevirtv1.RemoveVolumeOptions{Name: dvName}); err != nil { - return nil, status.Errorf(codes.Unavailable, - "failed to remove volume %s from VMI %s/%s: %v", dvName, w.infraNamespace, vmName, err) - } - if err := w.virtClient.EnsureVolumeRemoved(ctx, w.infraNamespace, vmName, dvName, 2*time.Minute); err != nil { - return nil, status.Errorf(codes.Unavailable, - "volume %s failed to be removed from VMI %s/%s in time: %v", dvName, w.infraNamespace, vmName, err) - } - - klog.V(3).Infof("Successfully unpublished volume %s from VMI %s/%s", dvName, w.infraNamespace, vmName) - return &csi.ControllerUnpublishVolumeResponse{}, nil -} - -// ControllerUnpublishVolume for NFS volumes (RWX Filesystem): deletes CiliumNetworkPolicy. -// For everything else (RWO and RWX Block), delegates to upstream so the hotplug -// DataVolume is removed from VM.spec.template.spec.volumes, then verifies the volume -// actually left VMI.Status.VolumeStatus. RWX Block volumes used +// ControllerUnpublishVolume for NFS volumes (RWX Filesystem): drops this node from the +// CiliumNetworkPolicy owners, which deletes the policy once no node is left. +// For everything else (RWO and RWX Block), delegates to upstream, which removes the +// hotplug DataVolume and waits for it to leave VMI.Status.VolumeStatus. RWX Block volumes used // for live migration must NOT be treated as NFS — their cleanup goes through upstream // hotplug removal, otherwise stale VM-spec entries permanently block re-attachment to // a different VM via the anti-split-brain check (issue #2634). @@ -485,7 +430,7 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context } if !isNFSVolume(pvc) { - return w.unpublishHotplugVolume(ctx, req) + return w.ControllerService.ControllerUnpublishVolume(ctx, req) } // NFS volume: remove VMI ownerReference from CiliumNetworkPolicy diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go index 80c941bb3..46e2e1dce 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go @@ -3,9 +3,12 @@ package main import ( "context" "encoding/json" + "errors" + "flag" "net" "reflect" "testing" + "time" csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" @@ -15,8 +18,15 @@ import ( "google.golang.org/grpc/test/bufconn" "google.golang.org/protobuf/types/known/emptypb" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" kubevirtclient "kubevirt.io/csi-driver/pkg/kubevirt" @@ -32,7 +42,7 @@ type fakeVirtClient struct { } func (f *fakeVirtClient) GetDataVolume(_ context.Context, _, name string) (*cdiv1.DataVolume, error) { - return nil, errors.NewNotFound(cdiv1.SchemeGroupVersion.WithResource("datavolumes").GroupResource(), name) + return nil, k8serrors.NewNotFound(cdiv1.SchemeGroupVersion.WithResource("datavolumes").GroupResource(), name) } func (f *fakeVirtClient) CreateDataVolume(_ context.Context, _ string, dv *cdiv1.DataVolume) (*cdiv1.DataVolume, error) { @@ -43,7 +53,7 @@ func (f *fakeVirtClient) CreateDataVolume(_ context.Context, _ string, dv *cdiv1 func newTestControllerService(virtClient kubevirtclient.Client) *WrappedControllerService { enforcement := util.StorageClassEnforcement{AllowAll: true} return &WrappedControllerService{ - ControllerService: service.NewControllerService(virtClient, "infra-ns", nil, enforcement), + ControllerService: service.NewControllerService(virtClient, "infra-ns", nil, enforcement, true), virtClient: virtClient, infraNamespace: "infra-ns", storageClassEnforcement: enforcement, @@ -293,3 +303,186 @@ func TestBuildEndpointSelectorListsEveryVM(t *testing.T) { t.Errorf("buildEndpointSelector = %#v, want %#v", got, want) } } + +const ( + testInfraNamespace = "tenant-test" + testVMName = "worker-0" + testVolumeName = "pvc-0" +) + +// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its +// VMI still reports the hot-plugged volume. The VMI drops the volume once +// RemoveVolumeFromVMI is called, or when the garbage collector deletes a +// VMI whose controlling VM is gone. +type teardownClient struct { + kubevirtclient.Client + vmiOwners []metav1.OwnerReference + vmiRemovals int + lookups int +} + +func (c *teardownClient) vmiGarbageCollected() bool { + owner := metav1.GetControllerOf(&metav1.ObjectMeta{OwnerReferences: c.vmiOwners}) + return owner != nil && owner.Kind == "VirtualMachine" && owner.APIVersion == kubevirtv1.SchemeGroupVersion.String() +} + +func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) { + c.lookups++ + return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName) +} + +func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) { + c.lookups++ + vmi := &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners}, + } + if c.vmiRemovals == 0 { + vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}} + } + return vmi, nil +} + +func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) { + return c.vmiRemovals > 0, nil +} + +func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error { + c.vmiRemovals++ + return nil +} + +func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error { + if c.vmiRemovals > 0 || c.vmiGarbageCollected() { + return nil + } + return errors.New("timed out waiting for the VMI to drop the volume") +} + +func vmOwner() []metav1.OwnerReference { + return []metav1.OwnerReference{ + *metav1.NewControllerRef(&kubevirtv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{Name: testVMName}}, kubevirtv1.VirtualMachineGroupVersionKind), + } +} + +// The flag has to reach the upstream controller service, which alone decides +// whether to call the VMI-level removevolume. A VMI no VM controls gets the +// call either way, since virt-api accepts it there. +func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) { + const name = "enable-vmi-hotplug-fallback" + tests := []struct { + name string + args []string + owners []metav1.OwnerReference + wantRemovals int + }{ + {name: "default, VM-owned VMI", args: nil, owners: vmOwner(), wantRemovals: 1}, + {name: "disabled, VM-owned VMI", args: []string{"--" + name + "=false"}, owners: vmOwner(), wantRemovals: 0}, + {name: "disabled, orphaned VMI", args: []string{"--" + name + "=false"}, owners: nil, wantRemovals: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Cleanup(func() { + _ = flag.Set(name, flag.Lookup(name).DefValue) + _ = flag.Set("infra-cluster-namespace", flag.Lookup("infra-cluster-namespace").DefValue) + }) + args := append([]string{"--infra-cluster-namespace=" + testInfraNamespace}, tt.args...) + if err := flag.CommandLine.Parse(args); err != nil { + t.Fatalf("parse flags: %v", err) + } + + client := &teardownClient{vmiOwners: tt.owners} + infra := fake.NewClientset(&corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testVolumeName, Namespace: testInfraNamespace}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}}, + }) + enforcement := util.StorageClassEnforcement{AllowAll: true, AllowDefault: true} + w := newWrappedControllerService(service.NewKubevirtCSIDriver(), client, infra, nil, nil, enforcement) + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName} + if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil { + t.Fatalf("ControllerUnpublishVolume() error = %v", err) + } + if client.vmiRemovals != tt.wantRemovals { + t.Errorf("RemoveVolumeFromVMI calls = %d, want %d", client.vmiRemovals, tt.wantRemovals) + } + }) + } +} + +func nfsCNP(owners ...string) *unstructured.Unstructured { + refs := make([]interface{}, 0, len(owners)) + for _, owner := range owners { + refs = append(refs, map[string]interface{}{ + "apiVersion": kubevirtv1.SchemeGroupVersion.String(), "kind": "VirtualMachineInstance", "name": owner, "uid": owner, + }) + } + return &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "cilium.io/v2", + "kind": "CiliumNetworkPolicy", + "metadata": map[string]interface{}{"name": "csi-nfs-" + testVolumeName, "namespace": testInfraNamespace, "ownerReferences": refs}, + }} +} + +// An RWX filesystem volume is served over NFS, not hot-plugged, so unpublish +// only releases this node from the volume's CiliumNetworkPolicy and never +// reaches the upstream hot-unplug. +func TestControllerUnpublishVolumeNFSReleasesNetworkPolicyOnly(t *testing.T) { + tests := []struct { + name string + owners []string + wantOwners []string + }{ + {name: "other nodes still mount it", owners: []string{testVMName, "worker-1"}, wantOwners: []string{"worker-1"}}, + {name: "last node", owners: []string{testVMName}, wantOwners: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := &teardownClient{vmiOwners: vmOwner()} + filesystem := corev1.PersistentVolumeFilesystem + infra := fake.NewClientset(&corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: testVolumeName, Namespace: testInfraNamespace}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: &filesystem, + }, + }) + dynamic := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{ciliumNetworkPolicyGVR: "CiliumNetworkPolicyList"}, nfsCNP(tt.owners...)) + w := &WrappedControllerService{ + ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true), + infraClient: infra, + dynamicClient: dynamic, + virtClient: client, + infraNamespace: testInfraNamespace, + } + + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName} + if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil { + t.Fatalf("ControllerUnpublishVolume() error = %v", err) + } + if client.lookups != 0 || client.vmiRemovals != 0 { + t.Errorf("VM/VMI lookups = %d, RemoveVolumeFromVMI calls = %d, want none", client.lookups, client.vmiRemovals) + } + + cnp, err := dynamic.Resource(ciliumNetworkPolicyGVR).Namespace(testInfraNamespace).Get(context.Background(), "csi-nfs-"+testVolumeName, metav1.GetOptions{}) + if tt.wantOwners == nil { + if !k8serrors.IsNotFound(err) { + t.Fatalf("CiliumNetworkPolicy still present after its last owner left: err = %v", err) + } + return + } + if err != nil { + t.Fatalf("get CiliumNetworkPolicy: %v", err) + } + var got []string + for _, ref := range cnp.GetOwnerReferences() { + got = append(got, ref.Name) + } + if len(got) != len(tt.wantOwners) || got[0] != tt.wantOwners[0] { + t.Errorf("CiliumNetworkPolicy owners = %v, want %v", got, tt.wantOwners) + } + }) + } +} diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/go.mod b/packages/apps/kubernetes/images/kubevirt-csi-driver/go.mod index e4852ac05..5f717fd67 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/go.mod +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/go.mod @@ -104,5 +104,5 @@ replace ( k8s.io/mount-utils => k8s.io/mount-utils v0.36.0 k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.36.0 k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.36.0 - kubevirt.io/csi-driver => github.com/kubevirt/csi-driver v0.0.0-20260803174826-f5de10d39556 + kubevirt.io/csi-driver => github.com/kubevirt/csi-driver v0.0.0-20260924151015-cb4770e4e475 ) diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/go.sum b/packages/apps/kubernetes/images/kubevirt-csi-driver/go.sum index 96a5c1a9e..c9fe98dee 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/go.sum +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/go.sum @@ -148,8 +148,8 @@ github.com/kubernetes-csi/csi-lib-utils v0.24.0 h1:hpL5ecxtr07/DNIF9Qn/gbNG/ZlgM github.com/kubernetes-csi/csi-lib-utils v0.24.0/go.mod h1:JbvkvtWghDcVZnwQoSi6Np9ITwqN7+sqLiSsM9y4kRE= github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0 h1:qS4r4ljINLWKJ9m9Ge3Q3sGZ/eIoDVDT2RhAdQFHb1k= github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0/go.mod h1:oGXx2XTEzs9ikW2V6IC1dD8trgjRsS/Mvc2JRiC618Y= -github.com/kubevirt/csi-driver v0.0.0-20260803174826-f5de10d39556 h1:8hJJprxgOOvg17Gc5LFLSKpjr77/eh82KW6URo/uiFs= -github.com/kubevirt/csi-driver v0.0.0-20260803174826-f5de10d39556/go.mod h1:ErITd7C1TCf5N8MRWOjoabX5s5sfW2BUGOOiUOtx+9E= +github.com/kubevirt/csi-driver v0.0.0-20260924151015-cb4770e4e475 h1:82B2pAet/OdspUVsJ8VWl4KI58dfcyJJPyrr/dW6o08= +github.com/kubevirt/csi-driver v0.0.0-20260924151015-cb4770e4e475/go.mod h1:ErITd7C1TCf5N8MRWOjoabX5s5sfW2BUGOOiUOtx+9E= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/main.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/main.go index 0ff995e6d..fdd370a1b 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/main.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/main.go @@ -38,6 +38,8 @@ var ( runNodeService = flag.Bool("run-node-service", true, "Specifies rather or not to run the node service, the default is true") runControllerService = flag.Bool("run-controller-service", true, "Specifies rather or not to run the controller service, the default is true") + enableVMIHotplugFallback = flag.Bool("enable-vmi-hotplug-fallback", true, "Specifies whether or not to hot-unplug a volume from a VM-owned VMI when it is not in the VM spec. Requires the KubeVirt HotplugVolumes feature gate; set to false when that gate is off. The default is true") + kubeAPIQPS = flag.Float64("kube-api-qps", defaultKubeAPIQPS, "QPS limit for the infra- and tenant-cluster Kubernetes API clients. client-go defaults to 5, which starves the once-per-second infra PVC-bound poll in the NFS ControllerPublishVolume path under a burst of concurrent attaches.") kubeAPIBurst = flag.Int("kube-api-burst", defaultKubeAPIBurst, "Burst limit for the infra- and tenant-cluster Kubernetes API clients (client-go default is 10).") ) @@ -153,14 +155,6 @@ func handle() { // Create upstream driver (provides Identity, Controller, Node services) upstreamDriver := service.NewKubevirtCSIDriver(). WithIdentityService(identityClientset) - if *runControllerService { - upstreamDriver = upstreamDriver.WithControllerService( - virtClient, - *infraClusterNamespace, - infraClusterLabelsMap, - storageClassEnforcement, - ) - } if *runNodeService { upstreamDriver = upstreamDriver.WithNodeService(nodeID) } @@ -176,15 +170,7 @@ func handle() { if err != nil { klog.Fatalf("Failed to build infra dynamic client: %v", err) } - cs = &WrappedControllerService{ - ControllerService: upstreamDriver.ControllerService, - infraClient: infraKubernetesClient, - dynamicClient: infraDynamicClient, - virtClient: virtClient, - infraNamespace: *infraClusterNamespace, - infraClusterLabels: infraClusterLabelsMap, - storageClassEnforcement: storageClassEnforcement, - } + cs = newWrappedControllerService(upstreamDriver, virtClient, infraKubernetesClient, infraDynamicClient, infraClusterLabelsMap, storageClassEnforcement) } var ns csi.NodeServer @@ -201,6 +187,34 @@ func handle() { s.Wait() } +// newWrappedControllerService reads the controller flags itself, so both the +// upstream service and the wrapper get them without an in-cluster config. +func newWrappedControllerService( + driver *service.KubevirtCSIDriver, + virtClient kubevirt.Client, + infraClient kubernetes.Interface, + dynamicClient dynamic.Interface, + infraClusterLabelsMap map[string]string, + storageClassEnforcement util.StorageClassEnforcement, +) *WrappedControllerService { + driver.WithControllerService( + virtClient, + *infraClusterNamespace, + infraClusterLabelsMap, + storageClassEnforcement, + *enableVMIHotplugFallback, + ) + return &WrappedControllerService{ + ControllerService: driver.ControllerService, + infraClient: infraClient, + dynamicClient: dynamicClient, + virtClient: virtClient, + infraNamespace: *infraClusterNamespace, + infraClusterLabels: infraClusterLabelsMap, + storageClassEnforcement: storageClassEnforcement, + } +} + // validateRateLimitFlags rejects non-positive limits. client-go treats QPS == 0 // as "use the 5 QPS default" and QPS < 0 as "no rate limiting at all", so an // operator passing 0 expecting "unlimited" would silently get the starving