From 1725226d2332efc2cf0085a77e4f4aa19f01a798 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 09:30:36 +0100 Subject: [PATCH 1/3] fix(linstor): restore snapshots on a schedulable node A volume restored from a snapshot is pinned to the one node it is restored on. linstor-csi picked that node among every ONLINE snapshot node: its preferred-node filter matched all nodes, so the node the scheduler selected got no priority, and a cordoned node was as good as any other. A pod whose volume landed on a cordoned node stayed Pending until the node was uncordoned, which made Velero data mover backups time out on every storage node cordoned ahead of maintenance, even when schedulable replicas held the same snapshot. Prefer the selected node when it holds the snapshot, and skip cordoned nodes while another node holds it. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../003-snapshot-restore-node-selection.diff | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff diff --git a/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff new file mode 100644 index 000000000..0ad54da77 --- /dev/null +++ b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff @@ -0,0 +1,249 @@ +diff --git a/cmd/linstor-csi/linstor-csi.go b/cmd/linstor-csi/linstor-csi.go +index bd28e06..5bed73b 100644 +--- a/cmd/linstor-csi/linstor-csi.go ++++ b/cmd/linstor-csi/linstor-csi.go +@@ -135,6 +135,7 @@ func main() { + client.LogOut(logOut), + client.PropertyNamespace(*propNs), + client.LabelBySP(*labelBySP), ++ client.ConfigureKubernetesIfAvailable(), + ) + if err != nil { + log.Fatal(err) +diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go +index 5d71c37..e8e67ec 100644 +--- a/pkg/client/linstor.go ++++ b/pkg/client/linstor.go +@@ -46,6 +46,8 @@ import ( + "github.com/pborman/uuid" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ++ apierrors "k8s.io/apimachinery/pkg/api/errors" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/mount-utils" + utilexec "k8s.io/utils/exec" + "k8s.io/utils/ptr" +@@ -72,6 +74,9 @@ type Linstor struct { + mounter mount.Interface + resizer *mount.ResizeFs + labelBySP bool ++ // nodeSchedulable reports whether new pods can be scheduled on a node. ++ // It is nil when not running in Kubernetes: every node then counts as schedulable. ++ nodeSchedulable func(ctx context.Context, node string) (bool, error) + } + + // NewLinstor returns a high-level linstor client for CSI applications to interact with +@@ -125,6 +130,34 @@ func APIClient(c *lc.HighLevelClient) func(*Linstor) error { + } + } + ++// ConfigureKubernetesIfAvailable lets the client read Kubernetes Node objects, so that a snapshot is ++// not restored on a cordoned node when another node holds it. Outside of Kubernetes it does nothing. ++func ConfigureKubernetesIfAvailable() func(*Linstor) error { ++ return func(l *Linstor) error { ++ cl, _, err := utils.KubernetesClient() ++ if err != nil { ++ // Not running in kubernetes ++ return nil ++ } ++ ++ l.nodeSchedulable = func(ctx context.Context, node string) (bool, error) { ++ n, err := cl.CoreV1().Nodes().Get(ctx, node, metav1.GetOptions{}) ++ if err != nil { ++ if apierrors.IsNotFound(err) { ++ // A LINSTOR node without a Kubernetes Node: nothing says it is cordoned. ++ return true, nil ++ } ++ ++ return true, err ++ } ++ ++ return !n.Spec.Unschedulable, nil ++ } ++ ++ return nil ++ } ++} ++ + // LogOut sets the Linstor client to write logs to the provided io.Writer + // instead of discarding logs. + func LogOut(out io.Writer) func(*Linstor) error { +@@ -1567,34 +1600,29 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu + return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) + } + +- // Collect available snapshot nodes, preferring those matching the topology hints. +- var preferred, available []string ++ // Collect the snapshot nodes LINSTOR can restore on, and whether Kubernetes can schedule pods there. ++ var available []string ++ schedulable := make(map[string]bool, len(snap.Nodes)) + for _, snapNode := range snap.Nodes { + if err := s.NodeAvailable(ctx, snapNode); err != nil { + logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") + continue + } + +- if slices.Contains(preferredNodes, snapNode) { +- preferred = append(preferred, snapNode) +- } +- + available = append(available, snapNode) +- } ++ schedulable[snapNode] = true + +- // Pick a random node from preferred candidates, falling back to any available node. +- // Randomization distributes restore load across snapshot nodes, preventing all +- // clones of the same source from concentrating on a single node. +- candidates := preferred +- if len(candidates) == 0 { +- candidates = available +- } ++ if s.nodeSchedulable != nil { ++ ok, err := s.nodeSchedulable(ctx, snapNode) ++ if err != nil { ++ logger.WithField("selected node candidate", snapNode).WithError(err).Warn("could not check if node is cordoned, assuming it is not") ++ } + +- var selectedNode string +- if len(candidates) > 0 { +- selectedNode = candidates[rand.Intn(len(candidates))] ++ schedulable[snapNode] = ok ++ } + } + ++ selectedNode := selectRestoreNode(available, preferredNodes, func(node string) bool { return schedulable[node] }) + if selectedNode == "" { + return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name) + } +@@ -1893,6 +1921,39 @@ func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) e + return nil + } + ++// selectRestoreNode picks the node a snapshot is restored on, among the available nodes holding it. ++// ++// The restored volume is pinned to that node, so the consumer pod has to run there: ++// - preferredNodes[0] is the node the scheduler selected for a WaitForFirstConsumer volume. If it holds the ++// snapshot, restoring there lets the pod start without another scheduling round. ++// - Otherwise a random node is picked, which spreads restores of the same source across its snapshot nodes. ++// Cordoned nodes are skipped as long as another node holds the snapshot: a pod can never be scheduled on ++// them, so a volume restored there stays unusable until the node is uncordoned. ++// ++// Only nodes that are also in preferredNodes are considered, unless none is. ++func selectRestoreNode(available, preferredNodes []string, schedulable func(node string) bool) string { ++ candidates := slices.DeleteFunc(slices.Clone(available), func(node string) bool { ++ return !slices.Contains(preferredNodes, node) ++ }) ++ if len(candidates) == 0 { ++ candidates = available ++ } ++ ++ if usable := slices.DeleteFunc(slices.Clone(candidates), func(node string) bool { return !schedulable(node) }); len(usable) > 0 { ++ candidates = usable ++ } ++ ++ if len(candidates) == 0 { ++ return "" ++ } ++ ++ if len(preferredNodes) > 0 && slices.Contains(candidates, preferredNodes[0]) { ++ return preferredNodes[0] ++ } ++ ++ return candidates[rand.Intn(len(candidates))] ++} ++ + // FindSnapsByID searches the snapshot in the backend + func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { + snapshotId, err := volume.ParseSnapshotId(id) +diff --git a/pkg/client/linstor_test.go b/pkg/client/linstor_test.go +index 4f3405b..84cfca2 100644 +--- a/pkg/client/linstor_test.go ++++ b/pkg/client/linstor_test.go +@@ -577,3 +577,87 @@ func TestGetSnapshotRemoteAndReadiness(t *testing.T) { + }) + } + } ++ ++func TestLinstor_reconcileSnapshotResourcesSkipsCordonedNodes(t *testing.T) { ++ // A replicated volume has a snapshot on three nodes, one of them cordoned. The scheduler selected a node ++ // without the snapshot. Restoring on the cordoned node pins the new volume to a node no pod can be scheduled ++ // on, so the restore must always go to one of the other two. ++ const ( ++ sourceRD = "pvc-source" ++ snapName = "snapshot-1" ++ targetRD = "pvc-target" ++ cordoned = "node-3" ++ ) ++ ++ snapNodes := []string{"node-1", "node-2", cordoned} ++ preferred := []string{"node-9", "node-1", "node-2", cordoned} ++ ++ restoredOn := map[string]int{} ++ ++ for range 50 { ++ res := mocks.ResourceProvider{} ++ res.On("GetAll", mock.Anything, targetRD).Return([]lapi.Resource{}, nil) ++ res.On("GetSnapshot", mock.Anything, sourceRD, snapName).Return(lapi.Snapshot{Name: snapName, Nodes: snapNodes}, nil) ++ res.On("RestoreSnapshot", mock.Anything, sourceRD, snapName, mock.Anything).Run(func(args mock.Arguments) { ++ restoredOn[args.Get(3).(lapi.SnapshotRestore).Nodes[0]]++ ++ }).Return(nil) ++ ++ nodes := mocks.NodeProvider{} ++ for _, n := range snapNodes { ++ nodes.On("Get", mock.Anything, n).Return(lapi.Node{Name: n, ConnectionStatus: "ONLINE"}, nil) ++ } ++ ++ cl := Linstor{ ++ client: &lc.HighLevelClient{Client: &lapi.Client{Resources: &res, Nodes: &nodes}}, ++ log: logrus.WithField("test", t.Name()), ++ nodeSchedulable: func(_ context.Context, node string) (bool, error) { ++ return node != cordoned, nil ++ }, ++ } ++ ++ snap := &volume.Snapshot{SnapshotId: volume.SnapshotId{SourceName: sourceRD, SnapshotName: snapName}} ++ assert.NoError(t, cl.reconcileSnapshotResources(context.Background(), snap, targetRD, preferred)) ++ } ++ ++ assert.Zero(t, restoredOn[cordoned], "restored on the cordoned node: %v", restoredOn) ++ assert.Equal(t, 50, restoredOn["node-1"]+restoredOn["node-2"]) ++} ++ ++func TestSelectRestoreNode(t *testing.T) { ++ all := func(string) bool { return true } ++ notNode3 := func(n string) bool { return n != "node-3" } ++ ++ t.Run("selected node holds the snapshot", func(t *testing.T) { ++ for range 20 { ++ assert.Equal(t, "node-2", selectRestoreNode([]string{"node-1", "node-2", "node-3"}, []string{"node-2", "node-1", "node-3"}, all)) ++ } ++ }) ++ ++ t.Run("selected node is cordoned", func(t *testing.T) { ++ for range 20 { ++ assert.NotEqual(t, "node-3", selectRestoreNode([]string{"node-1", "node-3"}, []string{"node-3", "node-1"}, notNode3)) ++ } ++ }) ++ ++ t.Run("only cordoned nodes hold the snapshot", func(t *testing.T) { ++ assert.Equal(t, "node-3", selectRestoreNode([]string{"node-3"}, []string{"node-1", "node-3"}, notNode3)) ++ }) ++ ++ t.Run("snapshot nodes outside the preferred nodes are a fallback", func(t *testing.T) { ++ assert.Equal(t, "node-2", selectRestoreNode([]string{"node-1", "node-2"}, []string{"node-2"}, all)) ++ assert.Equal(t, "node-1", selectRestoreNode([]string{"node-1"}, []string{"node-2"}, all)) ++ }) ++ ++ t.Run("no available node", func(t *testing.T) { ++ assert.Empty(t, selectRestoreNode(nil, []string{"node-1"}, all)) ++ }) ++ ++ t.Run("restores are spread when the selected node lacks the snapshot", func(t *testing.T) { ++ seen := map[string]bool{} ++ for range 200 { ++ seen[selectRestoreNode([]string{"node-1", "node-2"}, []string{"node-9", "node-1", "node-2"}, all)] = true ++ } ++ ++ assert.Equal(t, map[string]bool{"node-1": true, "node-2": true}, seen) ++ }) ++} From c6711d6fe9fdeb4b83e7812823408b5d7c3075f8 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 09:43:20 +0100 Subject: [PATCH 2/3] fix(linstor): prefer a schedulable snapshot node over topology hints The preferred nodes come from the requisite topology, which may leave out a snapshot node. Filtering on them first kept a cordoned node when the only schedulable snapshot node was outside that list, so the restored volume could still be unusable. Schedulability now decides before the topology hints, which only break ties. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../003-snapshot-restore-node-selection.diff | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff index 0ad54da77..75351fe8e 100644 --- a/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff +++ b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff @@ -11,7 +11,7 @@ index bd28e06..5bed73b 100644 if err != nil { log.Fatal(err) diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index 5d71c37..e8e67ec 100644 +index 5d71c37..43129c9 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go @@ -46,6 +46,8 @@ import ( @@ -115,7 +115,7 @@ index 5d71c37..e8e67ec 100644 if selectedNode == "" { return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name) } -@@ -1893,6 +1921,39 @@ func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) e +@@ -1893,6 +1921,43 @@ func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) e return nil } @@ -128,17 +128,21 @@ index 5d71c37..e8e67ec 100644 +// Cordoned nodes are skipped as long as another node holds the snapshot: a pod can never be scheduled on +// them, so a volume restored there stays unusable until the node is uncordoned. +// -+// Only nodes that are also in preferredNodes are considered, unless none is. ++// Nodes in preferredNodes come first, but a schedulable node outside them wins over a cordoned one inside them. +func selectRestoreNode(available, preferredNodes []string, schedulable func(node string) bool) string { -+ candidates := slices.DeleteFunc(slices.Clone(available), func(node string) bool { -+ return !slices.Contains(preferredNodes, node) -+ }) -+ if len(candidates) == 0 { -+ candidates = available ++ isPreferred := func(node string) bool { return slices.Contains(preferredNodes, node) } ++ both := func(node string) bool { return isPreferred(node) && schedulable(node) } ++ ++ var candidates []string ++ for _, keep := range []func(string) bool{both, schedulable, isPreferred} { ++ candidates = slices.DeleteFunc(slices.Clone(available), func(node string) bool { return !keep(node) }) ++ if len(candidates) > 0 { ++ break ++ } + } + -+ if usable := slices.DeleteFunc(slices.Clone(candidates), func(node string) bool { return !schedulable(node) }); len(usable) > 0 { -+ candidates = usable ++ if len(candidates) == 0 { ++ candidates = available + } + + if len(candidates) == 0 { @@ -156,10 +160,10 @@ index 5d71c37..e8e67ec 100644 func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { snapshotId, err := volume.ParseSnapshotId(id) diff --git a/pkg/client/linstor_test.go b/pkg/client/linstor_test.go -index 4f3405b..84cfca2 100644 +index 4f3405b..9dedb3c 100644 --- a/pkg/client/linstor_test.go +++ b/pkg/client/linstor_test.go -@@ -577,3 +577,87 @@ func TestGetSnapshotRemoteAndReadiness(t *testing.T) { +@@ -577,3 +577,93 @@ func TestGetSnapshotRemoteAndReadiness(t *testing.T) { }) } } @@ -225,6 +229,12 @@ index 4f3405b..84cfca2 100644 + } + }) + ++ t.Run("schedulable node outside the preferred nodes wins over a cordoned preferred node", func(t *testing.T) { ++ for range 20 { ++ assert.Equal(t, "node-2", selectRestoreNode([]string{"node-3", "node-2"}, []string{"node-3"}, notNode3)) ++ } ++ }) ++ + t.Run("only cordoned nodes hold the snapshot", func(t *testing.T) { + assert.Equal(t, "node-3", selectRestoreNode([]string{"node-3"}, []string{"node-1", "node-3"}, notNode3)) + }) From f7ee616f6cdcd9cf9828026b84a8df83cf7a83fd Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 17:27:48 +0100 Subject: [PATCH 3/3] fix(linstor): rebase the snapshot restore patch on linstor-csi v1.13.2 The import block of pkg/client/linstor.go changed in v1.13.2, so the patch no longer applied and the linstor-csi image failed to build. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../003-snapshot-restore-node-selection.diff | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff index 75351fe8e..7f96ed5f9 100644 --- a/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff +++ b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff @@ -1,8 +1,8 @@ diff --git a/cmd/linstor-csi/linstor-csi.go b/cmd/linstor-csi/linstor-csi.go -index bd28e06..5bed73b 100644 +index e260d7b..2062787 100644 --- a/cmd/linstor-csi/linstor-csi.go +++ b/cmd/linstor-csi/linstor-csi.go -@@ -135,6 +135,7 @@ func main() { +@@ -148,6 +148,7 @@ func main() { client.LogOut(logOut), client.PropertyNamespace(*propNs), client.LabelBySP(*labelBySP), @@ -11,7 +11,7 @@ index bd28e06..5bed73b 100644 if err != nil { log.Fatal(err) diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index 5d71c37..43129c9 100644 +index 5b52dd4..82f10d0 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go @@ -46,6 +46,8 @@ import ( @@ -20,10 +20,10 @@ index 5d71c37..43129c9 100644 "golang.org/x/sys/unix" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/mount-utils" utilexec "k8s.io/utils/exec" - "k8s.io/utils/ptr" -@@ -72,6 +74,9 @@ type Linstor struct { +@@ -73,6 +75,9 @@ type Linstor struct { mounter mount.Interface resizer *mount.ResizeFs labelBySP bool @@ -33,7 +33,7 @@ index 5d71c37..43129c9 100644 } // NewLinstor returns a high-level linstor client for CSI applications to interact with -@@ -125,6 +130,34 @@ func APIClient(c *lc.HighLevelClient) func(*Linstor) error { +@@ -126,6 +131,34 @@ func APIClient(c *lc.HighLevelClient) func(*Linstor) error { } } @@ -68,7 +68,7 @@ index 5d71c37..43129c9 100644 // LogOut sets the Linstor client to write logs to the provided io.Writer // instead of discarding logs. func LogOut(out io.Writer) func(*Linstor) error { -@@ -1567,34 +1600,29 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1834,34 +1867,29 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) } @@ -115,7 +115,7 @@ index 5d71c37..43129c9 100644 if selectedNode == "" { return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name) } -@@ -1893,6 +1921,43 @@ func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) e +@@ -2168,6 +2196,43 @@ func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) e return nil } @@ -160,10 +160,10 @@ index 5d71c37..43129c9 100644 func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { snapshotId, err := volume.ParseSnapshotId(id) diff --git a/pkg/client/linstor_test.go b/pkg/client/linstor_test.go -index 4f3405b..9dedb3c 100644 +index 54c5242..8da6cad 100644 --- a/pkg/client/linstor_test.go +++ b/pkg/client/linstor_test.go -@@ -577,3 +577,93 @@ func TestGetSnapshotRemoteAndReadiness(t *testing.T) { +@@ -994,3 +994,93 @@ func TestReconcileResourceDefinitionRGConflict(t *testing.T) { }) } }