diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/node.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/node.go index 7bac6f967..1e6eb1010 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/node.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/node.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "io/fs" "os" "os/exec" "path/filepath" "strings" + "sync" "time" csi "github.com/container-storage-interface/spec/lib/go/csi" @@ -25,6 +27,16 @@ var _ csi.NodeServer = &WrappedNodeService{} type WrappedNodeService struct { *service.NodeService mounter mount.Interface + // Target paths with an NFS publish running. kubelet retries a publish + // that outlives its 2m deadline while the first call still runs, and on + // a dead server every retry would block the same way: the soft temp mount + // gives up only after minutes, and the stat of a target that is already + // hard-mounted does not give up at all. + publishing sync.Map + // Replaced in tests, which cannot get the soft temp mount's EIO locally; + // nil means the os function. + lstat func(string) (os.FileInfo, error) + mkdir func(string, os.FileMode) error } // NodeStageVolume for NFS volumes is a no-op (NFS doesn't need staging). @@ -53,7 +65,19 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod return nil, status.Errorf(codes.Internal, "failed to parse NFS export: %v", err) } + lstat, mkdir := w.lstat, w.mkdir + if lstat == nil { + lstat = os.Lstat + } + if mkdir == nil { + mkdir = os.Mkdir + } + targetPath := req.GetTargetPath() + if _, busy := w.publishing.LoadOrStore(targetPath, struct{}{}); busy { + return nil, status.Errorf(codes.Aborted, "publish of NFS volume %s at %s already in progress", req.GetVolumeId(), targetPath) + } + defer w.publishing.Delete(targetPath) // Check if already mounted notMnt, err := w.mounter.IsLikelyNotMountPoint(targetPath) @@ -83,7 +107,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod // user files that were written before this fix (backward compatibility). tmpMount, err := os.MkdirTemp("", fmt.Sprintf("nfs-init-%s-", req.GetVolumeId())) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to create temp mount dir: %v", err) + return nil, status.Errorf(codes.Internal, "failed to create temp mount dir for volume %s: %v", req.GetVolumeId(), err) } rootSource := fmt.Sprintf("%s:%s", host, path) @@ -93,12 +117,21 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod // The price is a cache of its own too: pods already holding /data may not // see the entries the migration moves in until their attribute cache // expires (acdirmax). - rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache"} + // + // soft lets an RPC fail with EIO instead of blocking forever. That is + // safe here because this mount only does metadata work that a retried + // publish redoes, and it is why the volume mount below stays hard. Over + // NFSv4 a soft RPC fails only once the TCP connection is declared dead, + // which takes minutes with the default timeo and retrans. Those are not + // lowered: the first mount of a server sets them for the connection that + // every later mount of it on this node shares, the pods' hard mounts + // included. retry=0 stops mount.nfs from retrying a failed mount for 2m. + rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache", "soft", "retry=0"} if err := w.mounter.Mount(rootSource, tmpMount, "nfs", rootOpts); err != nil { if rmErr := os.Remove(tmpMount); rmErr != nil { klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, rmErr) } - return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err) + return nil, status.Errorf(codes.Internal, "NFS temp mount for volume %s failed: %v", req.GetVolumeId(), err) } defer func() { if err := unmountTempMount(w.mounter, tmpMount); err != nil { @@ -107,8 +140,10 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod }() dataDir := filepath.Join(tmpMount, "data") - if err := os.MkdirAll(dataDir, 0777); err != nil { - return nil, status.Errorf(codes.Internal, "failed to create /data subdir: %v", err) + // Not MkdirAll: with the server gone its stat fails on the mount point, + // and it then reports the local temp dir as existing instead of the EIO. + if err := mkdir(dataDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return nil, status.Errorf(codes.Internal, "failed to create /data subdir for volume %s: %v", req.GetVolumeId(), err) } // Auto-migrate: move user files from root into /data (skip internal artifacts). @@ -124,8 +159,14 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod } src := filepath.Join(tmpMount, name) dst := filepath.Join(dataDir, name) - if _, err := os.Stat(dst); err == nil { - continue // already exists in /data, skip + // Lstat: a user's symlink in /data counts as present whatever it + // points at, and is not replaced. + if _, err := lstat(dst); err == nil { + klog.Warningf("Not migrating %s for volume %s: /data/%s already exists, so %s stays in the export root", name, req.GetVolumeId(), name, name) + continue + } else if !os.IsNotExist(err) { + // The rename would replace whatever the failed stat could not see. + return nil, status.Errorf(codes.Internal, "failed to check /data/%s for volume %s: %v", name, req.GetVolumeId(), err) } klog.Infof("Migrating %s to /data/%s for volume %s", name, name, req.GetVolumeId()) if err := os.Rename(src, dst); err != nil { @@ -148,9 +189,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod const ( // Well inside the 2m kubelet puts on NodePublishVolume (csiTimeout in - // kubernetes pkg/volume/csi/csi_plugin.go). Bounds the unmount only: the - // temp mount is a hard mount, so the mount and the migration before it - // still block for as long as the server is gone. + // kubernetes pkg/volume/csi/csi_plugin.go). tempUnmountTimeout = 30 * time.Second // How long umount's output pipe may stay open once umount has exited or // been killed; the umount.nfs helper it forked can hold it. diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/node_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/node_test.go index 704b01021..9c932615d 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/node_test.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/node_test.go @@ -5,10 +5,15 @@ import ( "context" "os" "path/filepath" + "slices" "strings" + "sync/atomic" + "syscall" "testing" csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "k8s.io/klog/v2" mount "k8s.io/mount-utils" ) @@ -69,3 +74,267 @@ func TestNodePublishVolumeNFSLogsTempDirRemovalFailure(t *testing.T) { } t.Errorf("removal failure of %s was not logged as a warning; log output:\n%s", leftovers[0], logs.String()) } + +func nfsPublishRequest(target string) *csi.NodePublishVolumeRequest { + return &csi.NodePublishVolumeRequest{ + VolumeId: "pvc-1", + TargetPath: target, + PublishContext: map[string]string{nfsExportKey: "nfs://192.0.2.10:2049/export"}, + } +} + +func hasRetryTuning(opts []string) bool { + for _, o := range opts { + if o == "soft" || strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") || strings.HasPrefix(o, "retry=") { + return true + } + } + return false +} + +// A soft mount can report a failed write the server did apply, harmless for +// the metadata-only temp mount and not for the pods' data. +func TestNodePublishVolumeNFSMountsOnlyTheTempRootSoft(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + + // The fake drops the options of every mount an unmount leaves in place, so + // both sets are read while the temp mount is being unmounted, after the + // volume mount. + var tempOpts, finalOpts []string + m := mount.NewFakeMounter(nil) + m.UnmountFunc = func(path string) error { + for _, mp := range m.MountPoints { + switch { + case mp.Path == path: + tempOpts = mp.Opts + case mp.Device == "192.0.2.10:/export/data": + finalOpts = mp.Opts + } + } + return nil + } + w := &WrappedNodeService{mounter: m} + if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil { + t.Fatalf("NodePublishVolume: %v", err) + } + + for _, want := range []string{"soft", "retry=0"} { + if !slices.Contains(tempOpts, want) { + t.Errorf("temp mount options = %v, want %s", tempOpts, want) + } + } + // The first mount of a server sets the TCP timeouts of the connection the + // pods' mounts of it share. + for _, o := range tempOpts { + if strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") { + t.Errorf("temp mount options = %v, want the default timeo and retrans", tempOpts) + } + } + if finalOpts == nil { + t.Fatal("the /data subdir was not mounted before the temp mount was unmounted") + } + if hasRetryTuning(finalOpts) { + t.Errorf("volume mount options = %v, want a hard mount with default retry", finalOpts) + } +} + +// stalledMounter holds the first mount it gets until release is closed. +type stalledMounter struct { + *mount.FakeMounter + held atomic.Bool + entered chan struct{} + release chan struct{} +} + +func (b *stalledMounter) Mount(source, target, fstype string, opts []string) error { + if b.held.CompareAndSwap(false, true) { + close(b.entered) + <-b.release + } + return b.FakeMounter.Mount(source, target, fstype, opts) +} + +// kubelet retries a publish that outlives its deadline while the first call +// still runs; on a dead server each retry would block the same way. +func TestNodePublishVolumeNFSRefusesAPublishAlreadyInProgress(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + m := &stalledMounter{ + FakeMounter: mount.NewFakeMounter(nil), + entered: make(chan struct{}), + release: make(chan struct{}), + } + w := &WrappedNodeService{mounter: m} + target := filepath.Join(t.TempDir(), "target") + + first := make(chan error, 1) + go func() { + _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target)) + first <- err + }() + <-m.entered + + _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target)) + if status.Code(err) != codes.Aborted { + t.Errorf("publish while one is in progress: err = %v, want code Aborted", err) + } + if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "other"))); err != nil { + t.Errorf("publish of the same volume at another target: %v", err) + } + + close(m.release) + if err := <-first; err != nil { + t.Fatalf("first publish: %v", err) + } + if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target)); err != nil { + t.Errorf("publish after the first one finished: %v", err) + } +} + +func TestNodePublishVolumeNFSReleasesTheTargetAfterAFailedPublish(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + w := &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}} + req := nfsPublishRequest(filepath.Join(t.TempDir(), "target")) + for i := range 2 { + _, err := w.NodePublishVolume(context.Background(), req) + if code := status.Code(err); code != codes.Internal { + t.Fatalf("publish %d: err = %v, want code Internal from the failed mount", i+1, err) + } + } +} + +// seedingMounter fills the first mount's target with what seed creates there. +type seedingMounter struct { + *mount.FakeMounter + seed func(target string) error +} + +func (s *seedingMounter) Mount(source, target, fstype string, opts []string) error { + if seed := s.seed; seed != nil { + s.seed = nil + if err := seed(target); err != nil { + return err + } + } + return s.FakeMounter.Mount(source, target, fstype, opts) +} + +func seedRootAndData(root *string, link string) func(string) error { + return func(target string) error { + *root = target + if err := os.WriteFile(filepath.Join(target, "f"), nil, 0o600); err != nil { + return err + } + if err := os.Mkdir(filepath.Join(target, "data"), 0o750); err != nil { + return err + } + return os.Symlink(link, filepath.Join(target, "data", "f")) + } +} + +func assertNotMigrated(t *testing.T, root string) { + t.Helper() + if _, err := os.Stat(filepath.Join(root, "f")); err != nil { + t.Errorf("source was moved: %v", err) + } + if fi, err := os.Lstat(filepath.Join(root, "data", "f")); err != nil || fi.Mode()&os.ModeSymlink == 0 { + t.Errorf("destination was replaced: %v, %v", fi, err) + } +} + +// A soft temp mount answers a stat with EIO while the server is unreachable; +// a rename issued once it is back would replace the file the stat missed. +func TestNodePublishVolumeNFSDoesNotMigrateOverAnUncheckedDestination(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + var root string + w := &WrappedNodeService{ + mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")}, + lstat: func(string) (os.FileInfo, error) { return nil, syscall.EIO }, + } + + _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))) + if status.Code(err) != codes.Internal { + t.Fatalf("err = %v, want code Internal from the failed stat", err) + } + assertNotMigrated(t, root) +} + +// The symlinks are the user's: one that loops or dangles is still an entry +// in /data, neither a reason to fail the publish nor one to replace it. +func TestNodePublishVolumeNFSLeavesASymlinkInDataAlone(t *testing.T) { + for name, link := range map[string]string{"loop": "f", "dangling": "missing"} { + t.Run(name, func(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + var root string + w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, link)}} + if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil { + t.Fatalf("NodePublishVolume: %v", err) + } + assertNotMigrated(t, root) + }) + } +} + +// The root entry stays where the pod cannot see it, so the skip has to be +// findable by an operator. +func TestNodePublishVolumeNFSWarnsAboutAnEntryLeftInTheRoot(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + var logs bytes.Buffer + klog.LogToStderr(false) + klog.SetOutput(&logs) + t.Cleanup(func() { + klog.SetOutput(os.Stderr) + klog.LogToStderr(true) + }) + var root string + w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")}} + if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil { + t.Fatalf("NodePublishVolume: %v", err) + } + klog.Flush() + for _, line := range strings.Split(logs.String(), "\n") { + if strings.HasPrefix(line, "W") && strings.Contains(line, "volume pvc-1") && strings.Contains(line, "/data/f") { + return + } + } + t.Errorf("the skipped migration of f was not logged as a warning; log output:\n%s", logs.String()) +} + +func TestNodePublishVolumeNFSTempMountErrorsNameTheVolume(t *testing.T) { + cases := map[string]struct { + tmpdir func(t *testing.T) string + w *WrappedNodeService + }{ + "temp dir": { + tmpdir: func(t *testing.T) string { return filepath.Join(t.TempDir(), "missing") }, + w: &WrappedNodeService{mounter: mount.NewFakeMounter(nil)}, + }, + "mount": { + tmpdir: func(t *testing.T) string { return t.TempDir() }, + w: &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Setenv("TMPDIR", tc.tmpdir(t)) + _, err := tc.w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))) + if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "volume pvc-1") { + t.Errorf("err = %v, want code Internal naming volume pvc-1", err) + } + }) + } +} + +// On a dead server the stat inside MkdirAll fails on the temp mount point +// itself, and MkdirAll then reports the local temp dir as already existing +// instead of the errno the server's absence produced. +func TestNodePublishVolumeNFSReportsTheDataDirErrno(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + w := &WrappedNodeService{ + mounter: mount.NewFakeMounter(nil), + mkdir: func(string, os.FileMode) error { return syscall.EIO }, + } + _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))) + if err == nil || !strings.Contains(err.Error(), syscall.EIO.Error()) || !strings.Contains(err.Error(), "volume pvc-1") { + t.Errorf("err = %v, want the EIO from mkdir, naming volume pvc-1", err) + } +}