mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 14:02:56 +09:00
feat(linstor): update linstor-csi patches
Remove merged RWX validation patch and add new patch that includes: - Randomized node selection for snapshot restore - Relocate replicas to optimal nodes after clone and snapshot restore Co-Authored-By: Claude <noreply@anthropic.com> Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
This commit is contained in:
2 files changed
+355
-718
No files matched your search
+355
@@ -0,0 +1,355 @@
|
||||
diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
|
||||
index f544493..ebaa3d7 100644
|
||||
--- a/pkg/client/linstor.go
|
||||
+++ b/pkg/client/linstor.go
|
||||
@@ -181,7 +181,7 @@ func LogLevel(s string) func(*Linstor) error {
|
||||
func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, error) {
|
||||
var vols []volume.VolumeStatus
|
||||
|
||||
- resourcesByName := make(map[string][]lapi.Resource)
|
||||
+ resourcesByName := make(map[string][]lapi.ResourceWithVolumes)
|
||||
|
||||
resDefs, err := s.client.ResourceDefinitions.GetAll(ctx, lapi.RDGetAllRequest{WithVolumeDefinitions: true})
|
||||
if err != nil {
|
||||
@@ -194,7 +194,7 @@ func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus,
|
||||
}
|
||||
|
||||
for i := range allResources {
|
||||
- resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i].Resource)
|
||||
+ resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i])
|
||||
}
|
||||
|
||||
for _, rd := range resDefs {
|
||||
@@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu
|
||||
return err
|
||||
}
|
||||
|
||||
+ if params.RelocateAfterClone {
|
||||
+ logger.Debug("relocate resources to optimal nodes")
|
||||
+
|
||||
+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
|
||||
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
logger.Debug("reconcile extra properties")
|
||||
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1280,6 +1288,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v
|
||||
return err
|
||||
}
|
||||
|
||||
+ if params.RelocateAfterSnapshotRestore {
|
||||
+ logger.Debug("relocate resources to optimal nodes")
|
||||
+
|
||||
+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
|
||||
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
logger.Debug("reconcile extra properties")
|
||||
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1470,9 +1486,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
|
||||
return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name)
|
||||
}
|
||||
|
||||
- // Optimize the node we use to restore. It should be one of the preferred nodes, or just the first with a snapshot
|
||||
- // if no preferred nodes match.
|
||||
- var selectedNode string
|
||||
+ // Collect available snapshot nodes, preferring those matching the topology hints.
|
||||
+ var preferred, available []string
|
||||
for _, snapNode := range snap.Nodes {
|
||||
if err := s.NodeAvailable(ctx, snapNode); err != nil {
|
||||
logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available")
|
||||
@@ -1480,13 +1495,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
|
||||
}
|
||||
|
||||
if slices.Contains(preferredNodes, snapNode) {
|
||||
- // We found a perfect candidate.
|
||||
- selectedNode = snapNode
|
||||
- break
|
||||
- } else if selectedNode == "" {
|
||||
- // Set a fallback if we have no candidate yet.
|
||||
- selectedNode = snapNode
|
||||
+ preferred = append(preferred, snapNode)
|
||||
}
|
||||
+
|
||||
+ available = append(available, snapNode)
|
||||
+ }
|
||||
+
|
||||
+ // Pick a random node from preferred candidates, falling back to any available node.
|
||||
+ // Randomization distributes restore load across snapshot nodes, preventing all
|
||||
+ // clones of the same source from concentrating on a single node.
|
||||
+ candidates := preferred
|
||||
+ if len(candidates) == 0 {
|
||||
+ candidates = available
|
||||
+ }
|
||||
+
|
||||
+ var selectedNode string
|
||||
+ if len(candidates) > 0 {
|
||||
+ selectedNode = candidates[rand.Intn(len(candidates))]
|
||||
}
|
||||
|
||||
if selectedNode == "" {
|
||||
@@ -1679,6 +1704,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In
|
||||
return nil
|
||||
}
|
||||
|
||||
+const propertyRelocationTriggered = "Aux/csi-relocation-triggered"
|
||||
+
|
||||
+// relocateResources migrates replicas from their current nodes to the nodes that
|
||||
+// LINSTOR's autoplacer considers optimal. This is a best-effort, fire-and-forget
|
||||
+// operation: migrate-disk API is asynchronous and the volume remains usable on its
|
||||
+// current nodes even if relocation fails.
|
||||
+func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) error {
|
||||
+ logger := s.log.WithFields(logrus.Fields{
|
||||
+ "volume": volID,
|
||||
+ "resourceGroup": rgName,
|
||||
+ })
|
||||
+
|
||||
+ // Check if relocation was already triggered (idempotency guard).
|
||||
+ rd, err := s.client.ResourceDefinitions.Get(ctx, volID)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to get resource definition: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ if rd.Props[propertyRelocationTriggered] != "" {
|
||||
+ logger.Debug("relocation already triggered, skipping")
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ // Get current diskful nodes.
|
||||
+ resources, err := s.client.Resources.GetAll(ctx, volID)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to list resources: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ currentNodes := util.DeployedDiskfullyNodes(resources)
|
||||
+
|
||||
+ // Query optimal placement from LINSTOR autoplacer.
|
||||
+ sizeInfo, err := s.client.ResourceGroups.QuerySizeInfo(ctx, rgName, lapi.QuerySizeInfoRequest{})
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to query size info: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ if sizeInfo.SpaceInfo == nil || len(sizeInfo.SpaceInfo.NextSpawnResult) == 0 {
|
||||
+ logger.Debug("no spawn result from query-size-info, skipping relocation")
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ // Build a map of optimal node -> storage pool.
|
||||
+ optimalPools := make(map[string]string, len(sizeInfo.SpaceInfo.NextSpawnResult))
|
||||
+ for _, r := range sizeInfo.SpaceInfo.NextSpawnResult {
|
||||
+ optimalPools[r.NodeName] = r.StorPoolName
|
||||
+ }
|
||||
+
|
||||
+ // Compute nodes to remove (current but not optimal) and nodes to add (optimal but not current).
|
||||
+ var nodesToRemove, nodesToAdd []string
|
||||
+
|
||||
+ for _, node := range currentNodes {
|
||||
+ if _, ok := optimalPools[node]; !ok {
|
||||
+ nodesToRemove = append(nodesToRemove, node)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ for _, r := range sizeInfo.SpaceInfo.NextSpawnResult {
|
||||
+ if !slices.Contains(currentNodes, r.NodeName) {
|
||||
+ nodesToAdd = append(nodesToAdd, r.NodeName)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // Only migrate min(remove, add) pairs.
|
||||
+ pairs := min(len(nodesToRemove), len(nodesToAdd))
|
||||
+ if pairs == 0 {
|
||||
+ logger.Info("no relocation needed, placement is already optimal")
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ logger.Infof("relocating %d replica(s): %v -> %v", pairs, nodesToRemove[:pairs], nodesToAdd[:pairs])
|
||||
+
|
||||
+ // Mark relocation as triggered before starting migrations.
|
||||
+ err = s.client.ResourceDefinitions.Modify(ctx, volID, lapi.GenericPropsModify{
|
||||
+ OverrideProps: map[string]string{propertyRelocationTriggered: "true"},
|
||||
+ })
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("failed to set relocation property: %w", err)
|
||||
+ }
|
||||
+
|
||||
+ // Migrate each pair sequentially (LINSTOR serializes at the RD level anyway).
|
||||
+ for i := range pairs {
|
||||
+ fromNode := nodesToRemove[i]
|
||||
+ toNode := nodesToAdd[i]
|
||||
+ storPool := optimalPools[toNode]
|
||||
+
|
||||
+ logger := logger.WithFields(logrus.Fields{"from": fromNode, "to": toNode, "storagePool": storPool})
|
||||
+
|
||||
+ logger.Info("creating diskless resource on target node")
|
||||
+
|
||||
+ err := s.client.Resources.MakeAvailable(ctx, volID, toNode, lapi.ResourceMakeAvailable{})
|
||||
+ if err != nil {
|
||||
+ logger.WithError(err).Warn("failed to create diskless on target, skipping this pair")
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ logger.Info("initiating migrate-disk")
|
||||
+
|
||||
+ err = s.client.Resources.Migrate(ctx, volID, fromNode, toNode, storPool)
|
||||
+ if err != nil {
|
||||
+ logger.WithError(err).Warn("migrate-disk failed, skipping this pair")
|
||||
+ continue
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
// FindSnapsByID searches the snapshot in the backend
|
||||
func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) {
|
||||
snapshotId, err := volume.ParseSnapshotId(id)
|
||||
@@ -2173,7 +2306,7 @@ func (s *Linstor) Status(ctx context.Context, volId string) ([]string, *csi.Volu
|
||||
"volume": volId,
|
||||
}).Debug("getting assignments")
|
||||
|
||||
- ress, err := s.client.Resources.GetAll(ctx, volId)
|
||||
+ ress, err := s.client.Resources.GetResourceView(ctx, &lapi.ListOpts{Resource: []string{volId}})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to list resources for '%s': %w", volId, err)
|
||||
}
|
||||
@@ -2261,13 +2394,20 @@ func GetSnapshotRemoteAndReadiness(snap *lapi.Snapshot) (string, bool, error) {
|
||||
return "", slices.Contains(snap.Flags, lapiconsts.FlagSuccessful), nil
|
||||
}
|
||||
|
||||
-func NodesAndConditionFromResources(ress []lapi.Resource) ([]string, *csi.VolumeCondition) {
|
||||
+func NodesAndConditionFromResources(ress []lapi.ResourceWithVolumes) ([]string, *csi.VolumeCondition) {
|
||||
var allNodes, abnormalNodes []string
|
||||
|
||||
for i := range ress {
|
||||
res := &ress[i]
|
||||
|
||||
- allNodes = append(allNodes, res.NodeName)
|
||||
+ // A resource is a CSI publish target if any of its volumes were created
|
||||
+ // by ControllerPublishVolume, identified by the temporary-diskless-attach property.
|
||||
+ if slices.ContainsFunc(res.Volumes, func(v lapi.Volume) bool {
|
||||
+ createdFor, ok := v.Props[linstor.PropertyCreatedFor]
|
||||
+ return ok && createdFor == linstor.CreatedForTemporaryDisklessAttach
|
||||
+ }) {
|
||||
+ allNodes = append(allNodes, res.NodeName)
|
||||
+ }
|
||||
|
||||
if res.State == nil {
|
||||
abnormalNodes = append(abnormalNodes, res.NodeName)
|
||||
diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go
|
||||
index 39acd95..110c597 100644
|
||||
--- a/pkg/volume/parameter.go
|
||||
+++ b/pkg/volume/parameter.go
|
||||
@@ -50,6 +50,8 @@ const (
|
||||
nfsservicename
|
||||
nfssquash
|
||||
nfsrecoveryvolumesize
|
||||
+ relocateafterclone
|
||||
+ relocateaftersnapshotrestore
|
||||
)
|
||||
|
||||
// Parameters configuration for linstor volumes.
|
||||
@@ -118,6 +120,10 @@ type Parameters struct {
|
||||
// NfsRecoveryVolumeSize sets the volume size (in bytes) of the recovery volume used by the NFS server.
|
||||
// Defaults to 300MiB.
|
||||
NfsRecoveryVolumeBytes int64
|
||||
+ // RelocateAfterClone triggers asynchronous relocation of replicas to optimal nodes after cloning.
|
||||
+ RelocateAfterClone bool
|
||||
+ // RelocateAfterSnapshotRestore triggers asynchronous relocation of replicas to optimal nodes after snapshot restore.
|
||||
+ RelocateAfterSnapshotRestore bool
|
||||
}
|
||||
|
||||
const DefaultDisklessStoragePoolName = "DfltDisklessStorPool"
|
||||
@@ -136,10 +142,12 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
|
||||
Encryption: false,
|
||||
PlacementPolicy: topology.AutoPlaceTopology,
|
||||
Properties: make(map[string]string),
|
||||
- NfsConfigTemplatePath: "/etc/nfs-helper/default-config.tmpl",
|
||||
- NfsServiceName: "linstor-csi-nfs",
|
||||
- NfsSquash: "no_root_squash",
|
||||
- NfsRecoveryVolumeBytes: 300 * 1024 * 1024,
|
||||
+ NfsConfigTemplatePath: "/etc/nfs-helper/default-config.tmpl",
|
||||
+ NfsServiceName: "linstor-csi-nfs",
|
||||
+ NfsSquash: "no_root_squash",
|
||||
+ NfsRecoveryVolumeBytes: 300 * 1024 * 1024,
|
||||
+ RelocateAfterClone: true,
|
||||
+ RelocateAfterSnapshotRestore: true,
|
||||
}
|
||||
|
||||
for k, v := range params {
|
||||
@@ -287,6 +295,20 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
|
||||
}
|
||||
|
||||
p.NfsRecoveryVolumeBytes = s.Value()
|
||||
+ case relocateafterclone:
|
||||
+ val, err := strconv.ParseBool(v)
|
||||
+ if err != nil {
|
||||
+ return p, err
|
||||
+ }
|
||||
+
|
||||
+ p.RelocateAfterClone = val
|
||||
+ case relocateaftersnapshotrestore:
|
||||
+ val, err := strconv.ParseBool(v)
|
||||
+ if err != nil {
|
||||
+ return p, err
|
||||
+ }
|
||||
+
|
||||
+ p.RelocateAfterSnapshotRestore = val
|
||||
}
|
||||
}
|
||||
|
||||
diff --git a/pkg/volume/paramkey_enumer.go b/pkg/volume/paramkey_enumer.go
|
||||
index 5474963..1c2fc87 100644
|
||||
--- a/pkg/volume/paramkey_enumer.go
|
||||
+++ b/pkg/volume/paramkey_enumer.go
|
||||
@@ -7,11 +7,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
-const _paramKeyName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesize"
|
||||
+const _paramKeyName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesizerelocateafterclonerelocateaftersnapshotrestore"
|
||||
|
||||
-var _paramKeyIndex = [...]uint16{0, 23, 32, 42, 61, 80, 99, 109, 115, 124, 133, 141, 155, 170, 189, 203, 210, 221, 237, 250, 260, 273, 293, 310, 324, 333, 354}
|
||||
+var _paramKeyIndex = [...]uint16{0, 23, 32, 42, 61, 80, 99, 109, 115, 124, 133, 141, 155, 170, 189, 203, 210, 221, 237, 250, 260, 273, 293, 310, 324, 333, 354, 372, 400}
|
||||
|
||||
-const _paramKeyLowerName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesize"
|
||||
+const _paramKeyLowerName = "allowremotevolumeaccessautoplaceclientlistdisklessonremainingdisklessstoragepooldonotplacewithregexencryptionfsoptslayerlistmountoptsnodelistplacementcountplacementpolicyreplicasondifferentreplicasonsamesizekibstoragepoolpostmountxfsoptsresourcegroupusepvcnameoverprovisionxreplicasondifferentnfsconfigtemplatenfsservicenamenfssquashnfsrecoveryvolumesizerelocateafterclonerelocateaftersnapshotrestore"
|
||||
|
||||
func (i paramKey) String() string {
|
||||
if i < 0 || i >= paramKey(len(_paramKeyIndex)-1) {
|
||||
@@ -50,9 +50,11 @@ func _paramKeyNoOp() {
|
||||
_ = x[nfsservicename-(23)]
|
||||
_ = x[nfssquash-(24)]
|
||||
_ = x[nfsrecoveryvolumesize-(25)]
|
||||
+ _ = x[relocateafterclone-(26)]
|
||||
+ _ = x[relocateaftersnapshotrestore-(27)]
|
||||
}
|
||||
|
||||
-var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize}
|
||||
+var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize, relocateafterclone, relocateaftersnapshotrestore}
|
||||
|
||||
var _paramKeyNameToValueMap = map[string]paramKey{
|
||||
_paramKeyName[0:23]: allowremotevolumeaccess,
|
||||
@@ -107,6 +109,10 @@ var _paramKeyNameToValueMap = map[string]paramKey{
|
||||
_paramKeyLowerName[324:333]: nfssquash,
|
||||
_paramKeyName[333:354]: nfsrecoveryvolumesize,
|
||||
_paramKeyLowerName[333:354]: nfsrecoveryvolumesize,
|
||||
+ _paramKeyName[354:372]: relocateafterclone,
|
||||
+ _paramKeyLowerName[354:372]: relocateafterclone,
|
||||
+ _paramKeyName[372:400]: relocateaftersnapshotrestore,
|
||||
+ _paramKeyLowerName[372:400]: relocateaftersnapshotrestore,
|
||||
}
|
||||
|
||||
var _paramKeyNames = []string{
|
||||
@@ -136,6 +142,8 @@ var _paramKeyNames = []string{
|
||||
_paramKeyName[310:324],
|
||||
_paramKeyName[324:333],
|
||||
_paramKeyName[333:354],
|
||||
+ _paramKeyName[354:372],
|
||||
+ _paramKeyName[372:400],
|
||||
}
|
||||
|
||||
// paramKeyString retrieves an enum value from the enum constants string name.
|
||||
@@ -1,718 +0,0 @@
|
||||
diff --git a/cmd/linstor-csi/linstor-csi.go b/cmd/linstor-csi/linstor-csi.go
|
||||
index 143f6cee..bd28e06e 100644
|
||||
--- a/cmd/linstor-csi/linstor-csi.go
|
||||
+++ b/cmd/linstor-csi/linstor-csi.go
|
||||
@@ -41,22 +41,23 @@ import (
|
||||
|
||||
func main() {
|
||||
var (
|
||||
- lsEndpoint = flag.String("linstor-endpoint", "", "Controller API endpoint for LINSTOR")
|
||||
- lsSkipTLSVerification = flag.Bool("linstor-skip-tls-verification", false, "If true, do not verify tls")
|
||||
- csiEndpoint = flag.String("csi-endpoint", "unix:///var/lib/kubelet/plugins/linstor.csi.linbit.com/csi.sock", "CSI endpoint")
|
||||
- node = flag.String("node", "", "Node ID to pass to node service")
|
||||
- logLevel = flag.String("log-level", "info", "Enable debug log output. Choose from: panic, fatal, error, warn, info, debug")
|
||||
- rps = flag.Float64("linstor-api-requests-per-second", 0, "Maximum allowed number of LINSTOR API requests per second. Default: Unlimited")
|
||||
- burst = flag.Int("linstor-api-burst", 1, "Maximum number of API requests allowed before being limited by requests-per-second. Default: 1 (no bursting)")
|
||||
- bearerTokenFile = flag.String("bearer-token", "", "Read the bearer token from the given file and use it for authentication.")
|
||||
- propNs = flag.String("property-namespace", linstor.NamespcAuxiliary, "Limit the reported topology keys to properties from the given namespace.")
|
||||
- labelBySP = flag.Bool("label-by-storage-pool", true, "Set to false to disable labeling of nodes based on their configured storage pools.")
|
||||
- nodeCacheTimeout = flag.Duration("node-cache-timeout", 1*time.Minute, "Duration for which the results of node and storage pool related API responses should be cached.")
|
||||
- resourceCacheTimeout = flag.Duration("resource-cache-timeout", 30*time.Second, "Duration for which the results of resource related API responses should be cached.")
|
||||
- resyncAfter = flag.Duration("resync-after", 5*time.Minute, "Duration after which reconciliations (such as for VolumeSnapshotClasses) should be rerun. Set to 0 to disable.")
|
||||
- enableRWX = flag.Bool("enable-rwx", false, "Enable RWX support via NFS (requires running in Kubernetes).")
|
||||
- namespace = flag.String("nfs-service-namespace", "", "The namespace the NFS service is running in.")
|
||||
- reactorConfigMapName = flag.String("nfs-reactor-config-map-name", "linstor-csi-nfs-reactor-config", "Name of the config map used to store promoter configuration")
|
||||
+ lsEndpoint = flag.String("linstor-endpoint", "", "Controller API endpoint for LINSTOR")
|
||||
+ lsSkipTLSVerification = flag.Bool("linstor-skip-tls-verification", false, "If true, do not verify tls")
|
||||
+ csiEndpoint = flag.String("csi-endpoint", "unix:///var/lib/kubelet/plugins/linstor.csi.linbit.com/csi.sock", "CSI endpoint")
|
||||
+ node = flag.String("node", "", "Node ID to pass to node service")
|
||||
+ logLevel = flag.String("log-level", "info", "Enable debug log output. Choose from: panic, fatal, error, warn, info, debug")
|
||||
+ rps = flag.Float64("linstor-api-requests-per-second", 0, "Maximum allowed number of LINSTOR API requests per second. Default: Unlimited")
|
||||
+ burst = flag.Int("linstor-api-burst", 1, "Maximum number of API requests allowed before being limited by requests-per-second. Default: 1 (no bursting)")
|
||||
+ bearerTokenFile = flag.String("bearer-token", "", "Read the bearer token from the given file and use it for authentication.")
|
||||
+ propNs = flag.String("property-namespace", linstor.NamespcAuxiliary, "Limit the reported topology keys to properties from the given namespace.")
|
||||
+ labelBySP = flag.Bool("label-by-storage-pool", true, "Set to false to disable labeling of nodes based on their configured storage pools.")
|
||||
+ nodeCacheTimeout = flag.Duration("node-cache-timeout", 1*time.Minute, "Duration for which the results of node and storage pool related API responses should be cached.")
|
||||
+ resourceCacheTimeout = flag.Duration("resource-cache-timeout", 30*time.Second, "Duration for which the results of resource related API responses should be cached.")
|
||||
+ resyncAfter = flag.Duration("resync-after", 5*time.Minute, "Duration after which reconciliations (such as for VolumeSnapshotClasses) should be rerun. Set to 0 to disable.")
|
||||
+ enableRWX = flag.Bool("enable-rwx", false, "Enable RWX support via NFS (requires running in Kubernetes).")
|
||||
+ namespace = flag.String("nfs-service-namespace", "", "The namespace the NFS service is running in.")
|
||||
+ reactorConfigMapName = flag.String("nfs-reactor-config-map-name", "linstor-csi-nfs-reactor-config", "Name of the config map used to store promoter configuration")
|
||||
+ disableRWXBlockValidation = flag.Bool("disable-rwx-block-validation", false, "Disable KubeVirt VM ownership validation for RWX block volumes.")
|
||||
)
|
||||
|
||||
flag.Var(&volume.DefaultRemoteAccessPolicy, "default-remote-access-policy", "")
|
||||
@@ -169,6 +170,10 @@ func main() {
|
||||
opts = append(opts, driver.ConfigureRWX(*namespace, *reactorConfigMapName))
|
||||
}
|
||||
|
||||
+ if *disableRWXBlockValidation {
|
||||
+ opts = append(opts, driver.DisableRWXBlockValidation())
|
||||
+ }
|
||||
+
|
||||
drv, err := driver.NewDriver(opts...)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go
|
||||
index bea69a8b..a39674b6 100644
|
||||
--- a/pkg/driver/driver.go
|
||||
+++ b/pkg/driver/driver.go
|
||||
@@ -83,6 +83,8 @@ type Driver struct {
|
||||
topologyPrefix string
|
||||
// resyncAfter is the interval after which reconciliations should be retried
|
||||
resyncAfter time.Duration
|
||||
+ // disableRWXBlockValidation disables KubeVirt VM ownership validation for RWX block volumes
|
||||
+ disableRWXBlockValidation bool
|
||||
|
||||
// Embed for forward compatibility.
|
||||
csi.UnimplementedIdentityServer
|
||||
@@ -300,6 +302,17 @@ func ResyncAfter(resyncAfter time.Duration) func(*Driver) error {
|
||||
}
|
||||
}
|
||||
|
||||
+// DisableRWXBlockValidation disables the KubeVirt VM ownership validation for RWX block volumes.
|
||||
+// When disabled, the driver will not check if multiple pods using the same RWX block volume
|
||||
+// belong to the same VM. This may be needed in environments where the validation causes issues
|
||||
+// or when using RWX block volumes outside of KubeVirt.
|
||||
+func DisableRWXBlockValidation() func(*Driver) error {
|
||||
+ return func(d *Driver) error {
|
||||
+ d.disableRWXBlockValidation = true
|
||||
+ return nil
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
// GetPluginInfo https://github.com/container-storage-interface/spec/blob/v1.9.0/spec.md#getplugininfo
|
||||
func (d Driver) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) {
|
||||
return &csi.GetPluginInfoResponse{
|
||||
@@ -751,6 +764,14 @@ func (d Driver) ControllerPublishVolume(ctx context.Context, req *csi.Controller
|
||||
// ReadWriteMany block volume
|
||||
rwxBlock := req.VolumeCapability.AccessMode.GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER && req.VolumeCapability.GetBlock() != nil
|
||||
|
||||
+ // Validate RWX block attachment to prevent misuse of allow-two-primaries
|
||||
+ if rwxBlock && !d.disableRWXBlockValidation {
|
||||
+ if _, err := utils.ValidateRWXBlockAttachment(ctx, d.kubeClient, d.log, req.GetVolumeId()); err != nil {
|
||||
+ return nil, status.Errorf(codes.FailedPrecondition,
|
||||
+ "ControllerPublishVolume failed for %s: %v", req.GetVolumeId(), err)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
devPath, err := d.Assignments.Attach(ctx, req.GetVolumeId(), req.GetNodeId(), rwxBlock)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal,
|
||||
diff --git a/pkg/utils/rwx_validation.go b/pkg/utils/rwx_validation.go
|
||||
new file mode 100644
|
||||
index 00000000..9fe82768
|
||||
--- /dev/null
|
||||
+++ b/pkg/utils/rwx_validation.go
|
||||
@@ -0,0 +1,263 @@
|
||||
+/*
|
||||
+CSI Driver for Linstor
|
||||
+Copyright © 2018 LINBIT USA, LLC
|
||||
+
|
||||
+This program is free software; you can redistribute it and/or modify
|
||||
+it under the terms of the GNU General Public License as published by
|
||||
+the Free Software Foundation; either version 2 of the License, or
|
||||
+(at your option) any later version.
|
||||
+
|
||||
+This program is distributed in the hope that it will be useful,
|
||||
+but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+GNU General Public License for more details.
|
||||
+
|
||||
+You should have received a copy of the GNU General Public License
|
||||
+along with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||
+*/
|
||||
+
|
||||
+package utils
|
||||
+
|
||||
+import (
|
||||
+ "context"
|
||||
+ "fmt"
|
||||
+
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
+ "k8s.io/apimachinery/pkg/runtime/schema"
|
||||
+ "k8s.io/client-go/dynamic"
|
||||
+)
|
||||
+
|
||||
+// KubeVirtVMLabel is the label that KubeVirt adds to pods to identify the VM they belong to.
|
||||
+const KubeVirtVMLabel = "vm.kubevirt.io/name"
|
||||
+
|
||||
+// KubeVirtHotplugDiskLabel is the label that KubeVirt adds to hotplug disk pods.
|
||||
+const KubeVirtHotplugDiskLabel = "kubevirt.io"
|
||||
+
|
||||
+// PodGVR is the GroupVersionResource for pods.
|
||||
+var PodGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
|
||||
+
|
||||
+// PVGVR is the GroupVersionResource for persistent volumes.
|
||||
+var PVGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "persistentvolumes"}
|
||||
+
|
||||
+// ValidateRWXBlockAttachment checks that RWX block volumes are only used by pods belonging to the same VM.
|
||||
+// This prevents misuse of allow-two-primaries while still permitting live migration.
|
||||
+// Returns the VM name if validation passes, or an error if:
|
||||
+// - Multiple pods from different VMs are trying to use the same volume
|
||||
+// - A pod without the KubeVirt VM label is trying to use a volume already attached elsewhere (strict mode)
|
||||
+// Returns empty string for VM name when no pods are using the volume or validation is skipped.
|
||||
+func ValidateRWXBlockAttachment(ctx context.Context, kubeClient dynamic.Interface, log *logrus.Entry, volumeID string) (string, error) {
|
||||
+ log.WithField("volumeID", volumeID).Info("validateRWXBlockAttachment called")
|
||||
+
|
||||
+ if kubeClient == nil {
|
||||
+ // Not running in Kubernetes, skip validation
|
||||
+ log.Warn("validateRWXBlockAttachment: kubeClient is nil, skipping validation")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Get PV to find PVC reference
|
||||
+ pv, err := kubeClient.Resource(PVGVR).Get(ctx, volumeID, metav1.GetOptions{})
|
||||
+ if err != nil {
|
||||
+ log.WithError(err).Warn("cannot validate RWX attachment: failed to get PV")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Verify that PV's volumeHandle matches the volumeID
|
||||
+ volumeHandle, found, err := unstructured.NestedString(pv.Object, "spec", "csi", "volumeHandle")
|
||||
+ if err != nil {
|
||||
+ log.WithError(err).Warnf("cannot validate RWX attachment: failed to read volumeHandle for PV %s", volumeID)
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ if !found {
|
||||
+ log.Warnf("cannot validate RWX attachment: volumeHandle not found for PV %s", volumeID)
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ if volumeHandle != volumeID {
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "volumeID": volumeID,
|
||||
+ "volumeHandle": volumeHandle,
|
||||
+ }).Warn("cannot validate RWX attachment: PV volumeHandle does not match volumeID")
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Extract claimRef from PV
|
||||
+ claimRef, found, _ := unstructured.NestedMap(pv.Object, "spec", "claimRef")
|
||||
+ if !found {
|
||||
+ log.Warn("cannot validate RWX attachment: PV has no claimRef")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ pvcName, _, _ := unstructured.NestedString(claimRef, "name")
|
||||
+ pvcNamespace, _, _ := unstructured.NestedString(claimRef, "namespace")
|
||||
+
|
||||
+ if pvcNamespace == "" || pvcName == "" {
|
||||
+ log.Warn("cannot validate RWX attachment: PVC name or namespace is empty in claimRef")
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // List all pods in the namespace
|
||||
+ podList, err := kubeClient.Resource(PodGVR).Namespace(pvcNamespace).List(ctx, metav1.ListOptions{})
|
||||
+ if err != nil {
|
||||
+ return "", fmt.Errorf("failed to list pods in namespace %s: %w", pvcNamespace, err)
|
||||
+ }
|
||||
+
|
||||
+ // Filter pods that use this PVC and are in a running/pending state
|
||||
+ type podInfo struct {
|
||||
+ name string
|
||||
+ vmName string
|
||||
+ }
|
||||
+
|
||||
+ var podsUsingPVC []podInfo
|
||||
+
|
||||
+ for _, item := range podList.Items {
|
||||
+ // Get pod phase from status
|
||||
+ phase, _, _ := unstructured.NestedString(item.Object, "status", "phase")
|
||||
+ if phase == "Succeeded" || phase == "Failed" {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ // Check if pod uses the PVC
|
||||
+ volumes, found, _ := unstructured.NestedSlice(item.Object, "spec", "volumes")
|
||||
+ if !found {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ for _, vol := range volumes {
|
||||
+ volMap, ok := vol.(map[string]interface{})
|
||||
+ if !ok {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ claimName, found, _ := unstructured.NestedString(volMap, "persistentVolumeClaim", "claimName")
|
||||
+ if !found || claimName != pvcName {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ // Extract VM name, handling both regular and hotplug disk pods
|
||||
+ vmName, err := GetVMNameFromPod(ctx, kubeClient, log, &item)
|
||||
+ if err != nil {
|
||||
+ log.WithError(err).WithField("pod", item.GetName()).Warn("failed to get VM name from pod")
|
||||
+ // Continue with empty vmName - will be caught by strict mode check
|
||||
+ vmName = ""
|
||||
+ }
|
||||
+
|
||||
+ podsUsingPVC = append(podsUsingPVC, podInfo{
|
||||
+ name: item.GetName(),
|
||||
+ vmName: vmName,
|
||||
+ })
|
||||
+
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // If 0 or 1 pod uses the PVC, no conflict possible
|
||||
+ if len(podsUsingPVC) <= 1 {
|
||||
+ // Return VM name if there's exactly one pod
|
||||
+ if len(podsUsingPVC) == 1 {
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "volumeID": volumeID,
|
||||
+ "vmName": podsUsingPVC[0].vmName,
|
||||
+ "podCount": 1,
|
||||
+ "pvcNamespace": pvcNamespace,
|
||||
+ "pvcName": pvcName,
|
||||
+ }).Info("validateRWXBlockAttachment: single pod found, returning VM name")
|
||||
+
|
||||
+ return podsUsingPVC[0].vmName, nil
|
||||
+ }
|
||||
+
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "volumeID": volumeID,
|
||||
+ "pvcNamespace": pvcNamespace,
|
||||
+ "pvcName": pvcName,
|
||||
+ }).Info("validateRWXBlockAttachment: no pods found using PVC")
|
||||
+
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Check that all pods belong to the same VM
|
||||
+ var vmName string
|
||||
+ for _, pod := range podsUsingPVC {
|
||||
+ if pod.vmName == "" {
|
||||
+ // Strict mode: if any pod doesn't have the KubeVirt label and there are multiple pods,
|
||||
+ // deny the attachment
|
||||
+ return "", fmt.Errorf("RWX block volume %s/%s is used by multiple pods but pod %s does not have the %s label; "+
|
||||
+ "RWX block volumes with allow-two-primaries are only supported for KubeVirt live migration",
|
||||
+ pvcNamespace, pvcName, pod.name, KubeVirtVMLabel)
|
||||
+ }
|
||||
+
|
||||
+ if vmName == "" {
|
||||
+ vmName = pod.vmName
|
||||
+ } else if vmName != pod.vmName {
|
||||
+ // Different VMs are trying to use the same volume
|
||||
+ return "", fmt.Errorf("RWX block volume %s/%s is being used by pods from different VMs (%s and %s); "+
|
||||
+ "this is not supported - RWX block volumes with allow-two-primaries are only for live migration of a single VM",
|
||||
+ pvcNamespace, pvcName, vmName, pod.vmName)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "pvcNamespace": pvcNamespace,
|
||||
+ "pvcName": pvcName,
|
||||
+ "vmName": vmName,
|
||||
+ "podCount": len(podsUsingPVC),
|
||||
+ }).Debug("RWX block attachment validated: all pods belong to the same VM (likely live migration)")
|
||||
+
|
||||
+ return vmName, nil
|
||||
+}
|
||||
+
|
||||
+// GetVMNameFromPod extracts the VM name from a pod, handling both regular virt-launcher pods
|
||||
+// and hotplug disk pods (which reference the virt-launcher pod via ownerReferences).
|
||||
+func GetVMNameFromPod(ctx context.Context, kubeClient dynamic.Interface, log *logrus.Entry, pod *unstructured.Unstructured) (string, error) {
|
||||
+ labels := pod.GetLabels()
|
||||
+ if labels == nil {
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ // Direct case: pod has vm.kubevirt.io/name label (virt-launcher pod)
|
||||
+ if vmName, ok := labels[KubeVirtVMLabel]; ok && vmName != "" {
|
||||
+ return vmName, nil
|
||||
+ }
|
||||
+
|
||||
+ // Hotplug disk case: pod has kubevirt.io: hotplug-disk label
|
||||
+ // Follow ownerReferences to find the virt-launcher pod
|
||||
+ if hotplugValue, ok := labels[KubeVirtHotplugDiskLabel]; ok && hotplugValue == "hotplug-disk" {
|
||||
+ ownerRefs := pod.GetOwnerReferences()
|
||||
+ for _, owner := range ownerRefs {
|
||||
+ if owner.Kind != "Pod" || owner.Controller == nil || !*owner.Controller {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ // Get the owner pod (virt-launcher)
|
||||
+ ownerPod, err := kubeClient.Resource(PodGVR).Namespace(pod.GetNamespace()).Get(ctx, owner.Name, metav1.GetOptions{})
|
||||
+ if err != nil {
|
||||
+ return "", fmt.Errorf("failed to get owner pod %s: %w", owner.Name, err)
|
||||
+ }
|
||||
+
|
||||
+ // Extract VM name from owner pod
|
||||
+ ownerLabels := ownerPod.GetLabels()
|
||||
+ if ownerLabels != nil {
|
||||
+ if vmName, ok := ownerLabels[KubeVirtVMLabel]; ok && vmName != "" {
|
||||
+ log.WithFields(logrus.Fields{
|
||||
+ "hotplugPod": pod.GetName(),
|
||||
+ "virtLauncher": owner.Name,
|
||||
+ "vmName": vmName,
|
||||
+ }).Debug("resolved VM name from hotplug disk pod via owner reference")
|
||||
+
|
||||
+ return vmName, nil
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return "", fmt.Errorf("owner pod %s does not have %s label", owner.Name, KubeVirtVMLabel)
|
||||
+ }
|
||||
+
|
||||
+ return "", fmt.Errorf("hotplug disk pod %s has no controller owner reference", pod.GetName())
|
||||
+ }
|
||||
+
|
||||
+ return "", nil
|
||||
+}
|
||||
diff --git a/pkg/utils/rwx_validation_test.go b/pkg/utils/rwx_validation_test.go
|
||||
new file mode 100644
|
||||
index 00000000..d75690f9
|
||||
--- /dev/null
|
||||
+++ b/pkg/utils/rwx_validation_test.go
|
||||
@@ -0,0 +1,342 @@
|
||||
+/*
|
||||
+CSI Driver for Linstor
|
||||
+Copyright © 2018 LINBIT USA, LLC
|
||||
+
|
||||
+This program is free software; you can redistribute it and/or modify
|
||||
+it under the terms of the GNU General Public License as published by
|
||||
+the Free Software Foundation; either version 2 of the License, or
|
||||
+(at your option) any later version.
|
||||
+
|
||||
+This program is distributed in the hope that it will be useful,
|
||||
+but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+GNU General Public License for more details.
|
||||
+
|
||||
+You should have received a copy of the GNU General Public License
|
||||
+along with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||
+*/
|
||||
+
|
||||
+package utils
|
||||
+
|
||||
+import (
|
||||
+ "context"
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+ "github.com/stretchr/testify/assert"
|
||||
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
+ "k8s.io/apimachinery/pkg/runtime"
|
||||
+ "k8s.io/apimachinery/pkg/runtime/schema"
|
||||
+ dynamicfake "k8s.io/client-go/dynamic/fake"
|
||||
+)
|
||||
+
|
||||
+func TestValidateRWXBlockAttachment(t *testing.T) {
|
||||
+ testCases := []struct {
|
||||
+ name string
|
||||
+ pods []*unstructured.Unstructured
|
||||
+ pvcName string
|
||||
+ namespace string
|
||||
+ expectError bool
|
||||
+ errorMsg string
|
||||
+ }{
|
||||
+ {
|
||||
+ name: "no pods using PVC",
|
||||
+ pods: []*unstructured.Unstructured{},
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "single pod using PVC",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "two pods same VM (live migration)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm1-xyz", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "two pods different VMs (should fail)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm2-xyz", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: true,
|
||||
+ errorMsg: "different VMs",
|
||||
+ },
|
||||
+ {
|
||||
+ name: "pod without KubeVirt label when multiple pods exist (strict mode)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "test-pvc", map[string]string{}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: true,
|
||||
+ errorMsg: "does not have the vm.kubevirt.io/name label",
|
||||
+ },
|
||||
+ {
|
||||
+ name: "completed pods should be ignored",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Succeeded"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "failed pods should be ignored",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Failed"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "pods in different namespace should not conflict",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "other", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "pods using different PVCs should not conflict",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("pod1", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("pod2", "default", "other-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "three pods from same VM (multi-node live migration scenario)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-a", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm1-b", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm1-c", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Pending"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "hotplug disk pod with virt-launcher (should succeed)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createHotplugDiskPod("hp-volume-xyz", "default", "test-pvc", "virt-launcher-vm1-abc", "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: false,
|
||||
+ },
|
||||
+ {
|
||||
+ name: "hotplug disks from different VMs (should fail)",
|
||||
+ pods: []*unstructured.Unstructured{
|
||||
+ createUnstructuredPod("virt-launcher-vm1-abc", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm1"}, "Running"),
|
||||
+ createHotplugDiskPod("hp-volume-vm1", "default", "test-pvc", "virt-launcher-vm1-abc", "Running"),
|
||||
+ createUnstructuredPod("virt-launcher-vm2-xyz", "default", "test-pvc", map[string]string{KubeVirtVMLabel: "vm2"}, "Running"),
|
||||
+ createHotplugDiskPod("hp-volume-vm2", "default", "test-pvc", "virt-launcher-vm2-xyz", "Running"),
|
||||
+ },
|
||||
+ pvcName: "test-pvc",
|
||||
+ namespace: "default",
|
||||
+ expectError: true,
|
||||
+ errorMsg: "different VMs",
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ for _, tc := range testCases {
|
||||
+ t.Run(tc.name, func(t *testing.T) {
|
||||
+ // Create fake dynamic client with test pods and PV
|
||||
+ scheme := runtime.NewScheme()
|
||||
+
|
||||
+ // Create PV object that references the PVC
|
||||
+ pv := createUnstructuredPV("test-volume-id", tc.namespace, tc.pvcName)
|
||||
+
|
||||
+ objects := make([]runtime.Object, 0, len(tc.pods)+1)
|
||||
+ objects = append(objects, pv)
|
||||
+
|
||||
+ for _, pod := range tc.pods {
|
||||
+ objects = append(objects, pod)
|
||||
+ }
|
||||
+
|
||||
+ gvrToListKind := map[schema.GroupVersionResource]string{
|
||||
+ PodGVR: "PodList",
|
||||
+ PVGVR: "PersistentVolumeList",
|
||||
+ }
|
||||
+ client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, objects...)
|
||||
+
|
||||
+ // Create logger
|
||||
+ logger := logrus.NewEntry(logrus.New())
|
||||
+ logger.Logger.SetLevel(logrus.DebugLevel)
|
||||
+
|
||||
+ // Run validation
|
||||
+ vmName, err := ValidateRWXBlockAttachment(context.Background(), client, logger, "test-volume-id")
|
||||
+
|
||||
+ if tc.expectError {
|
||||
+ assert.Error(t, err)
|
||||
+
|
||||
+ if tc.errorMsg != "" {
|
||||
+ assert.Contains(t, err.Error(), tc.errorMsg)
|
||||
+ }
|
||||
+ } else {
|
||||
+ assert.NoError(t, err)
|
||||
+ // VM name is returned when there are pods using the volume
|
||||
+ if len(tc.pods) > 0 {
|
||||
+ assert.NotEmpty(t, vmName)
|
||||
+ }
|
||||
+ }
|
||||
+ })
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func TestValidateRWXBlockAttachmentNoKubeClient(t *testing.T) {
|
||||
+ // When not running in Kubernetes (no client), validation should be skipped
|
||||
+ logger := logrus.NewEntry(logrus.New())
|
||||
+
|
||||
+ vmName, err := ValidateRWXBlockAttachment(context.Background(), nil, logger, "test-volume-id")
|
||||
+ assert.NoError(t, err)
|
||||
+ assert.Empty(t, vmName)
|
||||
+}
|
||||
+
|
||||
+func TestValidateRWXBlockAttachmentPVNotFound(t *testing.T) {
|
||||
+ // When PV is not found, validation should be skipped with warning
|
||||
+ scheme := runtime.NewScheme()
|
||||
+
|
||||
+ gvrToListKind := map[schema.GroupVersionResource]string{
|
||||
+ PodGVR: "PodList",
|
||||
+ PVGVR: "PersistentVolumeList",
|
||||
+ }
|
||||
+ client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind)
|
||||
+
|
||||
+ logger := logrus.NewEntry(logrus.New())
|
||||
+ logger.Logger.SetLevel(logrus.DebugLevel)
|
||||
+
|
||||
+ vmName, err := ValidateRWXBlockAttachment(context.Background(), client, logger, "non-existent-pv")
|
||||
+ assert.NoError(t, err)
|
||||
+ assert.Empty(t, vmName)
|
||||
+}
|
||||
+
|
||||
+// createUnstructuredPod creates an unstructured pod object for testing.
|
||||
+func createUnstructuredPod(name, namespace, pvcName string, labels map[string]string, phase string) *unstructured.Unstructured {
|
||||
+ pod := &unstructured.Unstructured{
|
||||
+ Object: map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "Pod",
|
||||
+ "metadata": map[string]interface{}{
|
||||
+ "name": name,
|
||||
+ "namespace": namespace,
|
||||
+ "labels": toStringInterfaceMap(labels),
|
||||
+ },
|
||||
+ "spec": map[string]interface{}{
|
||||
+ "volumes": []interface{}{
|
||||
+ map[string]interface{}{
|
||||
+ "name": "data",
|
||||
+ "persistentVolumeClaim": map[string]interface{}{
|
||||
+ "claimName": pvcName,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ "status": map[string]interface{}{
|
||||
+ "phase": phase,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ return pod
|
||||
+}
|
||||
+
|
||||
+// createUnstructuredPV creates an unstructured PersistentVolume object for testing.
|
||||
+func createUnstructuredPV(name, pvcNamespace, pvcName string) *unstructured.Unstructured {
|
||||
+ pv := &unstructured.Unstructured{
|
||||
+ Object: map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "PersistentVolume",
|
||||
+ "metadata": map[string]interface{}{
|
||||
+ "name": name,
|
||||
+ },
|
||||
+ "spec": map[string]interface{}{
|
||||
+ "claimRef": map[string]interface{}{
|
||||
+ "name": pvcName,
|
||||
+ "namespace": pvcNamespace,
|
||||
+ },
|
||||
+ "csi": map[string]interface{}{
|
||||
+ "volumeHandle": name,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ return pv
|
||||
+}
|
||||
+
|
||||
+// toStringInterfaceMap converts map[string]string to map[string]interface{}.
|
||||
+func toStringInterfaceMap(m map[string]string) map[string]interface{} {
|
||||
+ result := make(map[string]interface{})
|
||||
+
|
||||
+ for k, v := range m {
|
||||
+ result[k] = v
|
||||
+ }
|
||||
+
|
||||
+ return result
|
||||
+}
|
||||
+
|
||||
+// createHotplugDiskPod creates a hotplug disk pod that references a virt-launcher pod via ownerReferences.
|
||||
+func createHotplugDiskPod(name, namespace, pvcName, ownerPodName, phase string) *unstructured.Unstructured {
|
||||
+ pod := &unstructured.Unstructured{
|
||||
+ Object: map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "Pod",
|
||||
+ "metadata": map[string]interface{}{
|
||||
+ "name": name,
|
||||
+ "namespace": namespace,
|
||||
+ "labels": map[string]interface{}{
|
||||
+ "kubevirt.io": "hotplug-disk",
|
||||
+ },
|
||||
+ "ownerReferences": []interface{}{
|
||||
+ map[string]interface{}{
|
||||
+ "apiVersion": "v1",
|
||||
+ "kind": "Pod",
|
||||
+ "name": ownerPodName,
|
||||
+ "controller": true,
|
||||
+ "blockOwnerDeletion": true,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ "spec": map[string]interface{}{
|
||||
+ "volumes": []interface{}{
|
||||
+ map[string]interface{}{
|
||||
+ "name": "data",
|
||||
+ "persistentVolumeClaim": map[string]interface{}{
|
||||
+ "claimName": pvcName,
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ },
|
||||
+ "status": map[string]interface{}{
|
||||
+ "phase": phase,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ return pod
|
||||
+}
|
||||
Reference in new issue
Block a user