feat(kubernetes): honour --enable-vmi-hotplug-fallback in kubevirt-csi

kubevirt/csi-driver cb4770e4e475 adds the flag so the driver can run
against KubeVirt without the deprecated HotplugVolumes gate: with it
off, a volume missing from VM.spec is not unplugged through the
VMI-level removevolume on a VM-owned VMI, which virt-api rejects once
that gate is gone. Bump to it and hand the flag to the upstream
controller service. The flag defaults to true, which keeps the detach
path as it was. cb4770e4e475 is the previous pin plus that one change.

The wrapper's own hot-unplug override goes. It was a backport of
kubevirt/csi-driver#184, and the previous pin already contained that
change: upstream detaches from the VMI when the VM is gone and waits
for the VMI status to drop the volume before returning, so the
wrapper's re-check never found it attached. Keeping it would have
meant a second copy of the new ownership rule behind a branch nothing
reaches.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-09-28 23:25:37 +03:00
1 parent 88b61c96de
commit 350940dcda
5 files changed
+235 -83

No files matched your search

@@ -411,65 +411,10 @@ func (w *WrappedControllerService) ControllerPublishVolume(ctx context.Context,
}, nil
}
// unpublishHotplugVolume delegates to upstream for the hot-unplug, then re-verifies
// that the volume is gone from VMI.Status.VolumeStatus before returning success.
// Upstream treats a missing parent VM as "detach succeeded", but during a CAPI/CAPK
// teardown the VMI (and its hot-plug pod) can outlive the VM: the hot-plug pod then
// keeps the infra storage device exclusively attached to the source host, and every
// subsequent attach of the same volume fails (on DRBD with "failed to set source
// device readwrite"). When the VMI still reports the hot-plug after upstream
// returned success, detach it from the VMI directly. Backport of
// kubevirt/csi-driver#184; drop once it is merged and the module is bumped past it.
func (w *WrappedControllerService) unpublishHotplugVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
if _, err := w.ControllerService.ControllerUnpublishVolume(ctx, req); err != nil {
return nil, err
}
dvName := req.GetVolumeId()
_, vmName, err := cache.SplitMetaNamespaceKey(req.GetNodeId())
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to parse node ID %q: %v", req.GetNodeId(), err)
}
// GetVirtualMachine on this client returns a VirtualMachineInstance, not a VM.
vmi, err := w.virtClient.GetVirtualMachine(ctx, w.infraNamespace, vmName)
if err != nil {
if errors.IsNotFound(err) {
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
return nil, status.Errorf(codes.Unavailable,
"cannot verify VMI %s/%s after unpublish of %s: %v", w.infraNamespace, vmName, dvName, err)
}
stillAttached := false
for _, vs := range vmi.Status.VolumeStatus {
if vs.HotplugVolume != nil && vs.Name == dvName {
stillAttached = true
break
}
}
if !stillAttached {
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
klog.Infof("Volume %s still reported by VMI %s/%s after unpublish – detaching from the VMI", dvName, w.infraNamespace, vmName)
if err := w.virtClient.RemoveVolumeFromVMI(ctx, w.infraNamespace, vmName, &kubevirtv1.RemoveVolumeOptions{Name: dvName}); err != nil {
return nil, status.Errorf(codes.Unavailable,
"failed to remove volume %s from VMI %s/%s: %v", dvName, w.infraNamespace, vmName, err)
}
if err := w.virtClient.EnsureVolumeRemoved(ctx, w.infraNamespace, vmName, dvName, 2*time.Minute); err != nil {
return nil, status.Errorf(codes.Unavailable,
"volume %s failed to be removed from VMI %s/%s in time: %v", dvName, w.infraNamespace, vmName, err)
}
klog.V(3).Infof("Successfully unpublished volume %s from VMI %s/%s", dvName, w.infraNamespace, vmName)
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
// ControllerUnpublishVolume for NFS volumes (RWX Filesystem): deletes CiliumNetworkPolicy.
// For everything else (RWO and RWX Block), delegates to upstream so the hotplug
// DataVolume is removed from VM.spec.template.spec.volumes, then verifies the volume
// actually left VMI.Status.VolumeStatus. RWX Block volumes used
// ControllerUnpublishVolume for NFS volumes (RWX Filesystem): drops this node from the
// CiliumNetworkPolicy owners, which deletes the policy once no node is left.
// For everything else (RWO and RWX Block), delegates to upstream, which removes the
// hotplug DataVolume and waits for it to leave VMI.Status.VolumeStatus. RWX Block volumes used
// for live migration must NOT be treated as NFS — their cleanup goes through upstream
// hotplug removal, otherwise stale VM-spec entries permanently block re-attachment to
// a different VM via the anti-split-brain check (issue #2634).
@@ -485,7 +430,7 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context
}
if !isNFSVolume(pvc) {
return w.unpublishHotplugVolume(ctx, req)
return w.ControllerService.ControllerUnpublishVolume(ctx, req)
}
// NFS volume: remove VMI ownerReference from CiliumNetworkPolicy
@@ -3,9 +3,12 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"net"
"reflect"
"testing"
"time"
csi "github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc"
@@ -15,8 +18,15 @@ import (
"google.golang.org/grpc/test/bufconn"
"google.golang.org/protobuf/types/known/emptypb"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
dynamicfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
kubevirtv1 "kubevirt.io/api/core/v1"
cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
kubevirtclient "kubevirt.io/csi-driver/pkg/kubevirt"
@@ -32,7 +42,7 @@ type fakeVirtClient struct {
}
func (f *fakeVirtClient) GetDataVolume(_ context.Context, _, name string) (*cdiv1.DataVolume, error) {
return nil, errors.NewNotFound(cdiv1.SchemeGroupVersion.WithResource("datavolumes").GroupResource(), name)
return nil, k8serrors.NewNotFound(cdiv1.SchemeGroupVersion.WithResource("datavolumes").GroupResource(), name)
}
func (f *fakeVirtClient) CreateDataVolume(_ context.Context, _ string, dv *cdiv1.DataVolume) (*cdiv1.DataVolume, error) {
@@ -43,7 +53,7 @@ func (f *fakeVirtClient) CreateDataVolume(_ context.Context, _ string, dv *cdiv1
func newTestControllerService(virtClient kubevirtclient.Client) *WrappedControllerService {
enforcement := util.StorageClassEnforcement{AllowAll: true}
return &WrappedControllerService{
ControllerService: service.NewControllerService(virtClient, "infra-ns", nil, enforcement),
ControllerService: service.NewControllerService(virtClient, "infra-ns", nil, enforcement, true),
virtClient: virtClient,
infraNamespace: "infra-ns",
storageClassEnforcement: enforcement,
@@ -293,3 +303,186 @@ func TestBuildEndpointSelectorListsEveryVM(t *testing.T) {
t.Errorf("buildEndpointSelector = %#v, want %#v", got, want)
}
}
const (
testInfraNamespace = "tenant-test"
testVMName = "worker-0"
testVolumeName = "pvc-0"
)
// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its
// VMI still reports the hot-plugged volume. The VMI drops the volume once
// RemoveVolumeFromVMI is called, or when the garbage collector deletes a
// VMI whose controlling VM is gone.
type teardownClient struct {
kubevirtclient.Client
vmiOwners []metav1.OwnerReference
vmiRemovals int
lookups int
}
func (c *teardownClient) vmiGarbageCollected() bool {
owner := metav1.GetControllerOf(&metav1.ObjectMeta{OwnerReferences: c.vmiOwners})
return owner != nil && owner.Kind == "VirtualMachine" && owner.APIVersion == kubevirtv1.SchemeGroupVersion.String()
}
func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) {
c.lookups++
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName)
}
func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) {
c.lookups++
vmi := &kubevirtv1.VirtualMachineInstance{
ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners},
}
if c.vmiRemovals == 0 {
vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
}
return vmi, nil
}
func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) {
return c.vmiRemovals > 0, nil
}
func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error {
c.vmiRemovals++
return nil
}
func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error {
if c.vmiRemovals > 0 || c.vmiGarbageCollected() {
return nil
}
return errors.New("timed out waiting for the VMI to drop the volume")
}
func vmOwner() []metav1.OwnerReference {
return []metav1.OwnerReference{
*metav1.NewControllerRef(&kubevirtv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{Name: testVMName}}, kubevirtv1.VirtualMachineGroupVersionKind),
}
}
// The flag has to reach the upstream controller service, which alone decides
// whether to call the VMI-level removevolume. A VMI no VM controls gets the
// call either way, since virt-api accepts it there.
func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) {
const name = "enable-vmi-hotplug-fallback"
tests := []struct {
name string
args []string
owners []metav1.OwnerReference
wantRemovals int
}{
{name: "default, VM-owned VMI", args: nil, owners: vmOwner(), wantRemovals: 1},
{name: "disabled, VM-owned VMI", args: []string{"--" + name + "=false"}, owners: vmOwner(), wantRemovals: 0},
{name: "disabled, orphaned VMI", args: []string{"--" + name + "=false"}, owners: nil, wantRemovals: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Cleanup(func() {
_ = flag.Set(name, flag.Lookup(name).DefValue)
_ = flag.Set("infra-cluster-namespace", flag.Lookup("infra-cluster-namespace").DefValue)
})
args := append([]string{"--infra-cluster-namespace=" + testInfraNamespace}, tt.args...)
if err := flag.CommandLine.Parse(args); err != nil {
t.Fatalf("parse flags: %v", err)
}
client := &teardownClient{vmiOwners: tt.owners}
infra := fake.NewClientset(&corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: testVolumeName, Namespace: testInfraNamespace},
Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}},
})
enforcement := util.StorageClassEnforcement{AllowAll: true, AllowDefault: true}
w := newWrappedControllerService(service.NewKubevirtCSIDriver(), client, infra, nil, nil, enforcement)
req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName}
if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil {
t.Fatalf("ControllerUnpublishVolume() error = %v", err)
}
if client.vmiRemovals != tt.wantRemovals {
t.Errorf("RemoveVolumeFromVMI calls = %d, want %d", client.vmiRemovals, tt.wantRemovals)
}
})
}
}
func nfsCNP(owners ...string) *unstructured.Unstructured {
refs := make([]interface{}, 0, len(owners))
for _, owner := range owners {
refs = append(refs, map[string]interface{}{
"apiVersion": kubevirtv1.SchemeGroupVersion.String(), "kind": "VirtualMachineInstance", "name": owner, "uid": owner,
})
}
return &unstructured.Unstructured{Object: map[string]interface{}{
"apiVersion": "cilium.io/v2",
"kind": "CiliumNetworkPolicy",
"metadata": map[string]interface{}{"name": "csi-nfs-" + testVolumeName, "namespace": testInfraNamespace, "ownerReferences": refs},
}}
}
// An RWX filesystem volume is served over NFS, not hot-plugged, so unpublish
// only releases this node from the volume's CiliumNetworkPolicy and never
// reaches the upstream hot-unplug.
func TestControllerUnpublishVolumeNFSReleasesNetworkPolicyOnly(t *testing.T) {
tests := []struct {
name string
owners []string
wantOwners []string
}{
{name: "other nodes still mount it", owners: []string{testVMName, "worker-1"}, wantOwners: []string{"worker-1"}},
{name: "last node", owners: []string{testVMName}, wantOwners: nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &teardownClient{vmiOwners: vmOwner()}
filesystem := corev1.PersistentVolumeFilesystem
infra := fake.NewClientset(&corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: testVolumeName, Namespace: testInfraNamespace},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany},
VolumeMode: &filesystem,
},
})
dynamic := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(),
map[schema.GroupVersionResource]string{ciliumNetworkPolicyGVR: "CiliumNetworkPolicyList"}, nfsCNP(tt.owners...))
w := &WrappedControllerService{
ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true),
infraClient: infra,
dynamicClient: dynamic,
virtClient: client,
infraNamespace: testInfraNamespace,
}
req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName}
if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil {
t.Fatalf("ControllerUnpublishVolume() error = %v", err)
}
if client.lookups != 0 || client.vmiRemovals != 0 {
t.Errorf("VM/VMI lookups = %d, RemoveVolumeFromVMI calls = %d, want none", client.lookups, client.vmiRemovals)
}
cnp, err := dynamic.Resource(ciliumNetworkPolicyGVR).Namespace(testInfraNamespace).Get(context.Background(), "csi-nfs-"+testVolumeName, metav1.GetOptions{})
if tt.wantOwners == nil {
if !k8serrors.IsNotFound(err) {
t.Fatalf("CiliumNetworkPolicy still present after its last owner left: err = %v", err)
}
return
}
if err != nil {
t.Fatalf("get CiliumNetworkPolicy: %v", err)
}
var got []string
for _, ref := range cnp.GetOwnerReferences() {
got = append(got, ref.Name)
}
if len(got) != len(tt.wantOwners) || got[0] != tt.wantOwners[0] {
t.Errorf("CiliumNetworkPolicy owners = %v, want %v", got, tt.wantOwners)
}
})
}
}
@@ -104,5 +104,5 @@ replace (
k8s.io/mount-utils => k8s.io/mount-utils v0.36.0
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.36.0
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.36.0
kubevirt.io/csi-driver => github.com/kubevirt/csi-driver v0.0.0-20260803174826-f5de10d39556
kubevirt.io/csi-driver => github.com/kubevirt/csi-driver v0.0.0-20260924151015-cb4770e4e475
)
@@ -148,8 +148,8 @@ github.com/kubernetes-csi/csi-lib-utils v0.24.0 h1:hpL5ecxtr07/DNIF9Qn/gbNG/ZlgM
github.com/kubernetes-csi/csi-lib-utils v0.24.0/go.mod h1:JbvkvtWghDcVZnwQoSi6Np9ITwqN7+sqLiSsM9y4kRE=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0 h1:qS4r4ljINLWKJ9m9Ge3Q3sGZ/eIoDVDT2RhAdQFHb1k=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0/go.mod h1:oGXx2XTEzs9ikW2V6IC1dD8trgjRsS/Mvc2JRiC618Y=
github.com/kubevirt/csi-driver v0.0.0-20260803174826-f5de10d39556 h1:8hJJprxgOOvg17Gc5LFLSKpjr77/eh82KW6URo/uiFs=
github.com/kubevirt/csi-driver v0.0.0-20260803174826-f5de10d39556/go.mod h1:ErITd7C1TCf5N8MRWOjoabX5s5sfW2BUGOOiUOtx+9E=
github.com/kubevirt/csi-driver v0.0.0-20260924151015-cb4770e4e475 h1:82B2pAet/OdspUVsJ8VWl4KI58dfcyJJPyrr/dW6o08=
github.com/kubevirt/csi-driver v0.0.0-20260924151015-cb4770e4e475/go.mod h1:ErITd7C1TCf5N8MRWOjoabX5s5sfW2BUGOOiUOtx+9E=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@@ -38,6 +38,8 @@ var (
runNodeService = flag.Bool("run-node-service", true, "Specifies rather or not to run the node service, the default is true")
runControllerService = flag.Bool("run-controller-service", true, "Specifies rather or not to run the controller service, the default is true")
enableVMIHotplugFallback = flag.Bool("enable-vmi-hotplug-fallback", true, "Specifies whether or not to hot-unplug a volume from a VM-owned VMI when it is not in the VM spec. Requires the KubeVirt HotplugVolumes feature gate; set to false when that gate is off. The default is true")
kubeAPIQPS = flag.Float64("kube-api-qps", defaultKubeAPIQPS, "QPS limit for the infra- and tenant-cluster Kubernetes API clients. client-go defaults to 5, which starves the once-per-second infra PVC-bound poll in the NFS ControllerPublishVolume path under a burst of concurrent attaches.")
kubeAPIBurst = flag.Int("kube-api-burst", defaultKubeAPIBurst, "Burst limit for the infra- and tenant-cluster Kubernetes API clients (client-go default is 10).")
)
@@ -153,14 +155,6 @@ func handle() {
// Create upstream driver (provides Identity, Controller, Node services)
upstreamDriver := service.NewKubevirtCSIDriver().
WithIdentityService(identityClientset)
if *runControllerService {
upstreamDriver = upstreamDriver.WithControllerService(
virtClient,
*infraClusterNamespace,
infraClusterLabelsMap,
storageClassEnforcement,
)
}
if *runNodeService {
upstreamDriver = upstreamDriver.WithNodeService(nodeID)
}
@@ -176,15 +170,7 @@ func handle() {
if err != nil {
klog.Fatalf("Failed to build infra dynamic client: %v", err)
}
cs = &WrappedControllerService{
ControllerService: upstreamDriver.ControllerService,
infraClient: infraKubernetesClient,
dynamicClient: infraDynamicClient,
virtClient: virtClient,
infraNamespace: *infraClusterNamespace,
infraClusterLabels: infraClusterLabelsMap,
storageClassEnforcement: storageClassEnforcement,
}
cs = newWrappedControllerService(upstreamDriver, virtClient, infraKubernetesClient, infraDynamicClient, infraClusterLabelsMap, storageClassEnforcement)
}
var ns csi.NodeServer
@@ -201,6 +187,34 @@ func handle() {
s.Wait()
}
// newWrappedControllerService reads the controller flags itself, so both the
// upstream service and the wrapper get them without an in-cluster config.
func newWrappedControllerService(
driver *service.KubevirtCSIDriver,
virtClient kubevirt.Client,
infraClient kubernetes.Interface,
dynamicClient dynamic.Interface,
infraClusterLabelsMap map[string]string,
storageClassEnforcement util.StorageClassEnforcement,
) *WrappedControllerService {
driver.WithControllerService(
virtClient,
*infraClusterNamespace,
infraClusterLabelsMap,
storageClassEnforcement,
*enableVMIHotplugFallback,
)
return &WrappedControllerService{
ControllerService: driver.ControllerService,
infraClient: infraClient,
dynamicClient: dynamicClient,
virtClient: virtClient,
infraNamespace: *infraClusterNamespace,
infraClusterLabels: infraClusterLabelsMap,
storageClassEnforcement: storageClassEnforcement,
}
}
// validateRateLimitFlags rejects non-positive limits. client-go treats QPS == 0
// as "use the 5 QPS default" and QPS < 0 as "no rate limiting at all", so an
// operator passing 0 expecting "unlimited" would silently get the starving