fix(kubernetes): keep NFS publish from hanging on a dead server

The kubevirt-csi node plugin mounts the NFS export root into a temp
dir to create /data and migrate old files. That mount was hard, so
with the server gone the mkdir, readdir and rename calls on it never
returned. kubelet gives up after 2m and retries, and every retry
blocked the same way, piling up goroutines and nfs-init-* dirs.

Mount the temp root soft, so its RPCs fail once the connection to the
server is declared dead instead of blocking forever; it only does
metadata work that a retried publish redoes. retry=0 keeps mount.nfs
from retrying a failed temp mount for 2m. timeo and retrans keep their
defaults, because the first mount of a server sets them for the
connection the pods' mounts of it share. The volume mount the pod gets
stays hard, since a soft mount can report a failed write the server
applied.

Over NFSv4 that soft failure takes minutes, and a stat of a target
that is already hard-mounted can block for good, so a publish for a
target path that already has one running now returns Aborted instead
of starting another.

A soft stat can fail with EIO on a file /data already holds, and a
rename issued once the server is back would replace it, so the
migration now fails the publish on any stat error other than a
missing destination. It checks the destination with lstat, so a
user's symlink in /data that loops or dangles counts as present
instead of failing every publish or being replaced by the rename.

The skip it makes for a destination already in /data leaves the root
entry where the pod cannot see it, so it is now logged as a warning.
/data is created with a plain mkdir, since with the server gone
MkdirAll fails its stat on the mount point and then reports the local
temp dir as already existing instead of the EIO. The temp mount
errors now name the volume, like the migration errors do.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-10-10 20:46:12 +03:00
1 parent d7f3f0d392
commit f76dedf9ca
2 files changed
+318 -10

No files matched your search

@@ -4,10 +4,12 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
csi "github.com/container-storage-interface/spec/lib/go/csi"
@@ -25,6 +27,16 @@ var _ csi.NodeServer = &WrappedNodeService{}
type WrappedNodeService struct {
*service.NodeService
mounter mount.Interface
// Target paths with an NFS publish running. kubelet retries a publish
// that outlives its 2m deadline while the first call still runs, and on
// a dead server every retry would block the same way: the soft temp mount
// gives up only after minutes, and the stat of a target that is already
// hard-mounted does not give up at all.
publishing sync.Map
// Replaced in tests, which cannot get the soft temp mount's EIO locally;
// nil means the os function.
lstat func(string) (os.FileInfo, error)
mkdir func(string, os.FileMode) error
}
// NodeStageVolume for NFS volumes is a no-op (NFS doesn't need staging).
@@ -53,7 +65,19 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
return nil, status.Errorf(codes.Internal, "failed to parse NFS export: %v", err)
}
lstat, mkdir := w.lstat, w.mkdir
if lstat == nil {
lstat = os.Lstat
}
if mkdir == nil {
mkdir = os.Mkdir
}
targetPath := req.GetTargetPath()
if _, busy := w.publishing.LoadOrStore(targetPath, struct{}{}); busy {
return nil, status.Errorf(codes.Aborted, "publish of NFS volume %s at %s already in progress", req.GetVolumeId(), targetPath)
}
defer w.publishing.Delete(targetPath)
// Check if already mounted
notMnt, err := w.mounter.IsLikelyNotMountPoint(targetPath)
@@ -83,7 +107,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
// user files that were written before this fix (backward compatibility).
tmpMount, err := os.MkdirTemp("", fmt.Sprintf("nfs-init-%s-", req.GetVolumeId()))
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir: %v", err)
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir for volume %s: %v", req.GetVolumeId(), err)
}
rootSource := fmt.Sprintf("%s:%s", host, path)
@@ -93,12 +117,21 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
// The price is a cache of its own too: pods already holding /data may not
// see the entries the migration moves in until their attribute cache
// expires (acdirmax).
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache"}
//
// soft lets an RPC fail with EIO instead of blocking forever. That is
// safe here because this mount only does metadata work that a retried
// publish redoes, and it is why the volume mount below stays hard. Over
// NFSv4 a soft RPC fails only once the TCP connection is declared dead,
// which takes minutes with the default timeo and retrans. Those are not
// lowered: the first mount of a server sets them for the connection that
// every later mount of it on this node shares, the pods' hard mounts
// included. retry=0 stops mount.nfs from retrying a failed mount for 2m.
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache", "soft", "retry=0"}
if err := w.mounter.Mount(rootSource, tmpMount, "nfs", rootOpts); err != nil {
if rmErr := os.Remove(tmpMount); rmErr != nil {
klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, rmErr)
}
return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err)
return nil, status.Errorf(codes.Internal, "NFS temp mount for volume %s failed: %v", req.GetVolumeId(), err)
}
defer func() {
if err := unmountTempMount(w.mounter, tmpMount); err != nil {
@@ -107,8 +140,10 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
}()
dataDir := filepath.Join(tmpMount, "data")
if err := os.MkdirAll(dataDir, 0777); err != nil {
return nil, status.Errorf(codes.Internal, "failed to create /data subdir: %v", err)
// Not MkdirAll: with the server gone its stat fails on the mount point,
// and it then reports the local temp dir as existing instead of the EIO.
if err := mkdir(dataDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return nil, status.Errorf(codes.Internal, "failed to create /data subdir for volume %s: %v", req.GetVolumeId(), err)
}
// Auto-migrate: move user files from root into /data (skip internal artifacts).
@@ -124,8 +159,14 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
}
src := filepath.Join(tmpMount, name)
dst := filepath.Join(dataDir, name)
if _, err := os.Stat(dst); err == nil {
continue // already exists in /data, skip
// Lstat: a user's symlink in /data counts as present whatever it
// points at, and is not replaced.
if _, err := lstat(dst); err == nil {
klog.Warningf("Not migrating %s for volume %s: /data/%s already exists, so %s stays in the export root", name, req.GetVolumeId(), name, name)
continue
} else if !os.IsNotExist(err) {
// The rename would replace whatever the failed stat could not see.
return nil, status.Errorf(codes.Internal, "failed to check /data/%s for volume %s: %v", name, req.GetVolumeId(), err)
}
klog.Infof("Migrating %s to /data/%s for volume %s", name, name, req.GetVolumeId())
if err := os.Rename(src, dst); err != nil {
@@ -148,9 +189,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
const (
// Well inside the 2m kubelet puts on NodePublishVolume (csiTimeout in
// kubernetes pkg/volume/csi/csi_plugin.go). Bounds the unmount only: the
// temp mount is a hard mount, so the mount and the migration before it
// still block for as long as the server is gone.
// kubernetes pkg/volume/csi/csi_plugin.go).
tempUnmountTimeout = 30 * time.Second
// How long umount's output pipe may stay open once umount has exited or
// been killed; the umount.nfs helper it forked can hold it.
@@ -5,10 +5,15 @@ import (
"context"
"os"
"path/filepath"
"slices"
"strings"
"sync/atomic"
"syscall"
"testing"
csi "github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/klog/v2"
mount "k8s.io/mount-utils"
)
@@ -69,3 +74,267 @@ func TestNodePublishVolumeNFSLogsTempDirRemovalFailure(t *testing.T) {
}
t.Errorf("removal failure of %s was not logged as a warning; log output:\n%s", leftovers[0], logs.String())
}
func nfsPublishRequest(target string) *csi.NodePublishVolumeRequest {
return &csi.NodePublishVolumeRequest{
VolumeId: "pvc-1",
TargetPath: target,
PublishContext: map[string]string{nfsExportKey: "nfs://192.0.2.10:2049/export"},
}
}
func hasRetryTuning(opts []string) bool {
for _, o := range opts {
if o == "soft" || strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") || strings.HasPrefix(o, "retry=") {
return true
}
}
return false
}
// A soft mount can report a failed write the server did apply, harmless for
// the metadata-only temp mount and not for the pods' data.
func TestNodePublishVolumeNFSMountsOnlyTheTempRootSoft(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
// The fake drops the options of every mount an unmount leaves in place, so
// both sets are read while the temp mount is being unmounted, after the
// volume mount.
var tempOpts, finalOpts []string
m := mount.NewFakeMounter(nil)
m.UnmountFunc = func(path string) error {
for _, mp := range m.MountPoints {
switch {
case mp.Path == path:
tempOpts = mp.Opts
case mp.Device == "192.0.2.10:/export/data":
finalOpts = mp.Opts
}
}
return nil
}
w := &WrappedNodeService{mounter: m}
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
t.Fatalf("NodePublishVolume: %v", err)
}
for _, want := range []string{"soft", "retry=0"} {
if !slices.Contains(tempOpts, want) {
t.Errorf("temp mount options = %v, want %s", tempOpts, want)
}
}
// The first mount of a server sets the TCP timeouts of the connection the
// pods' mounts of it share.
for _, o := range tempOpts {
if strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") {
t.Errorf("temp mount options = %v, want the default timeo and retrans", tempOpts)
}
}
if finalOpts == nil {
t.Fatal("the /data subdir was not mounted before the temp mount was unmounted")
}
if hasRetryTuning(finalOpts) {
t.Errorf("volume mount options = %v, want a hard mount with default retry", finalOpts)
}
}
// stalledMounter holds the first mount it gets until release is closed.
type stalledMounter struct {
*mount.FakeMounter
held atomic.Bool
entered chan struct{}
release chan struct{}
}
func (b *stalledMounter) Mount(source, target, fstype string, opts []string) error {
if b.held.CompareAndSwap(false, true) {
close(b.entered)
<-b.release
}
return b.FakeMounter.Mount(source, target, fstype, opts)
}
// kubelet retries a publish that outlives its deadline while the first call
// still runs; on a dead server each retry would block the same way.
func TestNodePublishVolumeNFSRefusesAPublishAlreadyInProgress(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
m := &stalledMounter{
FakeMounter: mount.NewFakeMounter(nil),
entered: make(chan struct{}),
release: make(chan struct{}),
}
w := &WrappedNodeService{mounter: m}
target := filepath.Join(t.TempDir(), "target")
first := make(chan error, 1)
go func() {
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target))
first <- err
}()
<-m.entered
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target))
if status.Code(err) != codes.Aborted {
t.Errorf("publish while one is in progress: err = %v, want code Aborted", err)
}
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "other"))); err != nil {
t.Errorf("publish of the same volume at another target: %v", err)
}
close(m.release)
if err := <-first; err != nil {
t.Fatalf("first publish: %v", err)
}
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target)); err != nil {
t.Errorf("publish after the first one finished: %v", err)
}
}
func TestNodePublishVolumeNFSReleasesTheTargetAfterAFailedPublish(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
w := &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}}
req := nfsPublishRequest(filepath.Join(t.TempDir(), "target"))
for i := range 2 {
_, err := w.NodePublishVolume(context.Background(), req)
if code := status.Code(err); code != codes.Internal {
t.Fatalf("publish %d: err = %v, want code Internal from the failed mount", i+1, err)
}
}
}
// seedingMounter fills the first mount's target with what seed creates there.
type seedingMounter struct {
*mount.FakeMounter
seed func(target string) error
}
func (s *seedingMounter) Mount(source, target, fstype string, opts []string) error {
if seed := s.seed; seed != nil {
s.seed = nil
if err := seed(target); err != nil {
return err
}
}
return s.FakeMounter.Mount(source, target, fstype, opts)
}
func seedRootAndData(root *string, link string) func(string) error {
return func(target string) error {
*root = target
if err := os.WriteFile(filepath.Join(target, "f"), nil, 0o600); err != nil {
return err
}
if err := os.Mkdir(filepath.Join(target, "data"), 0o750); err != nil {
return err
}
return os.Symlink(link, filepath.Join(target, "data", "f"))
}
}
func assertNotMigrated(t *testing.T, root string) {
t.Helper()
if _, err := os.Stat(filepath.Join(root, "f")); err != nil {
t.Errorf("source was moved: %v", err)
}
if fi, err := os.Lstat(filepath.Join(root, "data", "f")); err != nil || fi.Mode()&os.ModeSymlink == 0 {
t.Errorf("destination was replaced: %v, %v", fi, err)
}
}
// A soft temp mount answers a stat with EIO while the server is unreachable;
// a rename issued once it is back would replace the file the stat missed.
func TestNodePublishVolumeNFSDoesNotMigrateOverAnUncheckedDestination(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
var root string
w := &WrappedNodeService{
mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")},
lstat: func(string) (os.FileInfo, error) { return nil, syscall.EIO },
}
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
if status.Code(err) != codes.Internal {
t.Fatalf("err = %v, want code Internal from the failed stat", err)
}
assertNotMigrated(t, root)
}
// The symlinks are the user's: one that loops or dangles is still an entry
// in /data, neither a reason to fail the publish nor one to replace it.
func TestNodePublishVolumeNFSLeavesASymlinkInDataAlone(t *testing.T) {
for name, link := range map[string]string{"loop": "f", "dangling": "missing"} {
t.Run(name, func(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
var root string
w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, link)}}
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
t.Fatalf("NodePublishVolume: %v", err)
}
assertNotMigrated(t, root)
})
}
}
// The root entry stays where the pod cannot see it, so the skip has to be
// findable by an operator.
func TestNodePublishVolumeNFSWarnsAboutAnEntryLeftInTheRoot(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
var logs bytes.Buffer
klog.LogToStderr(false)
klog.SetOutput(&logs)
t.Cleanup(func() {
klog.SetOutput(os.Stderr)
klog.LogToStderr(true)
})
var root string
w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")}}
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
t.Fatalf("NodePublishVolume: %v", err)
}
klog.Flush()
for _, line := range strings.Split(logs.String(), "\n") {
if strings.HasPrefix(line, "W") && strings.Contains(line, "volume pvc-1") && strings.Contains(line, "/data/f") {
return
}
}
t.Errorf("the skipped migration of f was not logged as a warning; log output:\n%s", logs.String())
}
func TestNodePublishVolumeNFSTempMountErrorsNameTheVolume(t *testing.T) {
cases := map[string]struct {
tmpdir func(t *testing.T) string
w *WrappedNodeService
}{
"temp dir": {
tmpdir: func(t *testing.T) string { return filepath.Join(t.TempDir(), "missing") },
w: &WrappedNodeService{mounter: mount.NewFakeMounter(nil)},
},
"mount": {
tmpdir: func(t *testing.T) string { return t.TempDir() },
w: &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}},
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
t.Setenv("TMPDIR", tc.tmpdir(t))
_, err := tc.w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "volume pvc-1") {
t.Errorf("err = %v, want code Internal naming volume pvc-1", err)
}
})
}
}
// On a dead server the stat inside MkdirAll fails on the temp mount point
// itself, and MkdirAll then reports the local temp dir as already existing
// instead of the errno the server's absence produced.
func TestNodePublishVolumeNFSReportsTheDataDirErrno(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
w := &WrappedNodeService{
mounter: mount.NewFakeMounter(nil),
mkdir: func(string, os.FileMode) error { return syscall.EIO },
}
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
if err == nil || !strings.Contains(err.Error(), syscall.EIO.Error()) || !strings.Contains(err.Error(), "volume pvc-1") {
t.Errorf("err = %v, want the EIO from mkdir, naming volume pvc-1", err)
}
}