mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
fix(kubernetes): keep NFS publish from hanging on a dead server
The kubevirt-csi node plugin mounts the NFS export root into a temp dir to create /data and migrate old files. That mount was hard, so with the server gone the mkdir, readdir and rename calls on it never returned. kubelet gives up after 2m and retries, and every retry blocked the same way, piling up goroutines and nfs-init-* dirs. Mount the temp root soft, so its RPCs fail once the connection to the server is declared dead instead of blocking forever; it only does metadata work that a retried publish redoes. retry=0 keeps mount.nfs from retrying a failed temp mount for 2m. timeo and retrans keep their defaults, because the first mount of a server sets them for the connection the pods' mounts of it share. The volume mount the pod gets stays hard, since a soft mount can report a failed write the server applied. Over NFSv4 that soft failure takes minutes, and a stat of a target that is already hard-mounted can block for good, so a publish for a target path that already has one running now returns Aborted instead of starting another. A soft stat can fail with EIO on a file /data already holds, and a rename issued once the server is back would replace it, so the migration now fails the publish on any stat error other than a missing destination. It checks the destination with lstat, so a user's symlink in /data that loops or dangles counts as present instead of failing every publish or being replaced by the rename. The skip it makes for a destination already in /data leaves the root entry where the pod cannot see it, so it is now logged as a warning. /data is created with a plain mkdir, since with the server gone MkdirAll fails its stat on the mount point and then reports the local temp dir as already existing instead of the EIO. The temp mount errors now name the volume, like the migration errors do. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
2 files changed
+318
-10
No files matched your search
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
csi "github.com/container-storage-interface/spec/lib/go/csi"
|
||||
@@ -25,6 +27,16 @@ var _ csi.NodeServer = &WrappedNodeService{}
|
||||
type WrappedNodeService struct {
|
||||
*service.NodeService
|
||||
mounter mount.Interface
|
||||
// Target paths with an NFS publish running. kubelet retries a publish
|
||||
// that outlives its 2m deadline while the first call still runs, and on
|
||||
// a dead server every retry would block the same way: the soft temp mount
|
||||
// gives up only after minutes, and the stat of a target that is already
|
||||
// hard-mounted does not give up at all.
|
||||
publishing sync.Map
|
||||
// Replaced in tests, which cannot get the soft temp mount's EIO locally;
|
||||
// nil means the os function.
|
||||
lstat func(string) (os.FileInfo, error)
|
||||
mkdir func(string, os.FileMode) error
|
||||
}
|
||||
|
||||
// NodeStageVolume for NFS volumes is a no-op (NFS doesn't need staging).
|
||||
@@ -53,7 +65,19 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
|
||||
return nil, status.Errorf(codes.Internal, "failed to parse NFS export: %v", err)
|
||||
}
|
||||
|
||||
lstat, mkdir := w.lstat, w.mkdir
|
||||
if lstat == nil {
|
||||
lstat = os.Lstat
|
||||
}
|
||||
if mkdir == nil {
|
||||
mkdir = os.Mkdir
|
||||
}
|
||||
|
||||
targetPath := req.GetTargetPath()
|
||||
if _, busy := w.publishing.LoadOrStore(targetPath, struct{}{}); busy {
|
||||
return nil, status.Errorf(codes.Aborted, "publish of NFS volume %s at %s already in progress", req.GetVolumeId(), targetPath)
|
||||
}
|
||||
defer w.publishing.Delete(targetPath)
|
||||
|
||||
// Check if already mounted
|
||||
notMnt, err := w.mounter.IsLikelyNotMountPoint(targetPath)
|
||||
@@ -83,7 +107,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
|
||||
// user files that were written before this fix (backward compatibility).
|
||||
tmpMount, err := os.MkdirTemp("", fmt.Sprintf("nfs-init-%s-", req.GetVolumeId()))
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir: %v", err)
|
||||
return nil, status.Errorf(codes.Internal, "failed to create temp mount dir for volume %s: %v", req.GetVolumeId(), err)
|
||||
}
|
||||
|
||||
rootSource := fmt.Sprintf("%s:%s", host, path)
|
||||
@@ -93,12 +117,21 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
|
||||
// The price is a cache of its own too: pods already holding /data may not
|
||||
// see the entries the migration moves in until their attribute cache
|
||||
// expires (acdirmax).
|
||||
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache"}
|
||||
//
|
||||
// soft lets an RPC fail with EIO instead of blocking forever. That is
|
||||
// safe here because this mount only does metadata work that a retried
|
||||
// publish redoes, and it is why the volume mount below stays hard. Over
|
||||
// NFSv4 a soft RPC fails only once the TCP connection is declared dead,
|
||||
// which takes minutes with the default timeo and retrans. Those are not
|
||||
// lowered: the first mount of a server sets them for the connection that
|
||||
// every later mount of it on this node shares, the pods' hard mounts
|
||||
// included. retry=0 stops mount.nfs from retrying a failed mount for 2m.
|
||||
rootOpts := []string{"nfsvers=4.2", fmt.Sprintf("port=%s", port), "nosharecache", "soft", "retry=0"}
|
||||
if err := w.mounter.Mount(rootSource, tmpMount, "nfs", rootOpts); err != nil {
|
||||
if rmErr := os.Remove(tmpMount); rmErr != nil {
|
||||
klog.Warningf("Failed to remove temp dir %s: %v", tmpMount, rmErr)
|
||||
}
|
||||
return nil, status.Errorf(codes.Internal, "NFS temp mount failed: %v", err)
|
||||
return nil, status.Errorf(codes.Internal, "NFS temp mount for volume %s failed: %v", req.GetVolumeId(), err)
|
||||
}
|
||||
defer func() {
|
||||
if err := unmountTempMount(w.mounter, tmpMount); err != nil {
|
||||
@@ -107,8 +140,10 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
|
||||
}()
|
||||
|
||||
dataDir := filepath.Join(tmpMount, "data")
|
||||
if err := os.MkdirAll(dataDir, 0777); err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to create /data subdir: %v", err)
|
||||
// Not MkdirAll: with the server gone its stat fails on the mount point,
|
||||
// and it then reports the local temp dir as existing instead of the EIO.
|
||||
if err := mkdir(dataDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return nil, status.Errorf(codes.Internal, "failed to create /data subdir for volume %s: %v", req.GetVolumeId(), err)
|
||||
}
|
||||
|
||||
// Auto-migrate: move user files from root into /data (skip internal artifacts).
|
||||
@@ -124,8 +159,14 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
|
||||
}
|
||||
src := filepath.Join(tmpMount, name)
|
||||
dst := filepath.Join(dataDir, name)
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
continue // already exists in /data, skip
|
||||
// Lstat: a user's symlink in /data counts as present whatever it
|
||||
// points at, and is not replaced.
|
||||
if _, err := lstat(dst); err == nil {
|
||||
klog.Warningf("Not migrating %s for volume %s: /data/%s already exists, so %s stays in the export root", name, req.GetVolumeId(), name, name)
|
||||
continue
|
||||
} else if !os.IsNotExist(err) {
|
||||
// The rename would replace whatever the failed stat could not see.
|
||||
return nil, status.Errorf(codes.Internal, "failed to check /data/%s for volume %s: %v", name, req.GetVolumeId(), err)
|
||||
}
|
||||
klog.Infof("Migrating %s to /data/%s for volume %s", name, name, req.GetVolumeId())
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
@@ -148,9 +189,7 @@ func (w *WrappedNodeService) NodePublishVolume(ctx context.Context, req *csi.Nod
|
||||
|
||||
const (
|
||||
// Well inside the 2m kubelet puts on NodePublishVolume (csiTimeout in
|
||||
// kubernetes pkg/volume/csi/csi_plugin.go). Bounds the unmount only: the
|
||||
// temp mount is a hard mount, so the mount and the migration before it
|
||||
// still block for as long as the server is gone.
|
||||
// kubernetes pkg/volume/csi/csi_plugin.go).
|
||||
tempUnmountTimeout = 30 * time.Second
|
||||
// How long umount's output pipe may stay open once umount has exited or
|
||||
// been killed; the umount.nfs helper it forked can hold it.
|
||||
|
||||
@@ -5,10 +5,15 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
csi "github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"k8s.io/klog/v2"
|
||||
mount "k8s.io/mount-utils"
|
||||
)
|
||||
@@ -69,3 +74,267 @@ func TestNodePublishVolumeNFSLogsTempDirRemovalFailure(t *testing.T) {
|
||||
}
|
||||
t.Errorf("removal failure of %s was not logged as a warning; log output:\n%s", leftovers[0], logs.String())
|
||||
}
|
||||
|
||||
func nfsPublishRequest(target string) *csi.NodePublishVolumeRequest {
|
||||
return &csi.NodePublishVolumeRequest{
|
||||
VolumeId: "pvc-1",
|
||||
TargetPath: target,
|
||||
PublishContext: map[string]string{nfsExportKey: "nfs://192.0.2.10:2049/export"},
|
||||
}
|
||||
}
|
||||
|
||||
func hasRetryTuning(opts []string) bool {
|
||||
for _, o := range opts {
|
||||
if o == "soft" || strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") || strings.HasPrefix(o, "retry=") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A soft mount can report a failed write the server did apply, harmless for
|
||||
// the metadata-only temp mount and not for the pods' data.
|
||||
func TestNodePublishVolumeNFSMountsOnlyTheTempRootSoft(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
|
||||
// The fake drops the options of every mount an unmount leaves in place, so
|
||||
// both sets are read while the temp mount is being unmounted, after the
|
||||
// volume mount.
|
||||
var tempOpts, finalOpts []string
|
||||
m := mount.NewFakeMounter(nil)
|
||||
m.UnmountFunc = func(path string) error {
|
||||
for _, mp := range m.MountPoints {
|
||||
switch {
|
||||
case mp.Path == path:
|
||||
tempOpts = mp.Opts
|
||||
case mp.Device == "192.0.2.10:/export/data":
|
||||
finalOpts = mp.Opts
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
w := &WrappedNodeService{mounter: m}
|
||||
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
|
||||
t.Fatalf("NodePublishVolume: %v", err)
|
||||
}
|
||||
|
||||
for _, want := range []string{"soft", "retry=0"} {
|
||||
if !slices.Contains(tempOpts, want) {
|
||||
t.Errorf("temp mount options = %v, want %s", tempOpts, want)
|
||||
}
|
||||
}
|
||||
// The first mount of a server sets the TCP timeouts of the connection the
|
||||
// pods' mounts of it share.
|
||||
for _, o := range tempOpts {
|
||||
if strings.HasPrefix(o, "timeo=") || strings.HasPrefix(o, "retrans=") {
|
||||
t.Errorf("temp mount options = %v, want the default timeo and retrans", tempOpts)
|
||||
}
|
||||
}
|
||||
if finalOpts == nil {
|
||||
t.Fatal("the /data subdir was not mounted before the temp mount was unmounted")
|
||||
}
|
||||
if hasRetryTuning(finalOpts) {
|
||||
t.Errorf("volume mount options = %v, want a hard mount with default retry", finalOpts)
|
||||
}
|
||||
}
|
||||
|
||||
// stalledMounter holds the first mount it gets until release is closed.
|
||||
type stalledMounter struct {
|
||||
*mount.FakeMounter
|
||||
held atomic.Bool
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (b *stalledMounter) Mount(source, target, fstype string, opts []string) error {
|
||||
if b.held.CompareAndSwap(false, true) {
|
||||
close(b.entered)
|
||||
<-b.release
|
||||
}
|
||||
return b.FakeMounter.Mount(source, target, fstype, opts)
|
||||
}
|
||||
|
||||
// kubelet retries a publish that outlives its deadline while the first call
|
||||
// still runs; on a dead server each retry would block the same way.
|
||||
func TestNodePublishVolumeNFSRefusesAPublishAlreadyInProgress(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
m := &stalledMounter{
|
||||
FakeMounter: mount.NewFakeMounter(nil),
|
||||
entered: make(chan struct{}),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
w := &WrappedNodeService{mounter: m}
|
||||
target := filepath.Join(t.TempDir(), "target")
|
||||
|
||||
first := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target))
|
||||
first <- err
|
||||
}()
|
||||
<-m.entered
|
||||
|
||||
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target))
|
||||
if status.Code(err) != codes.Aborted {
|
||||
t.Errorf("publish while one is in progress: err = %v, want code Aborted", err)
|
||||
}
|
||||
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "other"))); err != nil {
|
||||
t.Errorf("publish of the same volume at another target: %v", err)
|
||||
}
|
||||
|
||||
close(m.release)
|
||||
if err := <-first; err != nil {
|
||||
t.Fatalf("first publish: %v", err)
|
||||
}
|
||||
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(target)); err != nil {
|
||||
t.Errorf("publish after the first one finished: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodePublishVolumeNFSReleasesTheTargetAfterAFailedPublish(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
w := &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}}
|
||||
req := nfsPublishRequest(filepath.Join(t.TempDir(), "target"))
|
||||
for i := range 2 {
|
||||
_, err := w.NodePublishVolume(context.Background(), req)
|
||||
if code := status.Code(err); code != codes.Internal {
|
||||
t.Fatalf("publish %d: err = %v, want code Internal from the failed mount", i+1, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seedingMounter fills the first mount's target with what seed creates there.
|
||||
type seedingMounter struct {
|
||||
*mount.FakeMounter
|
||||
seed func(target string) error
|
||||
}
|
||||
|
||||
func (s *seedingMounter) Mount(source, target, fstype string, opts []string) error {
|
||||
if seed := s.seed; seed != nil {
|
||||
s.seed = nil
|
||||
if err := seed(target); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.FakeMounter.Mount(source, target, fstype, opts)
|
||||
}
|
||||
|
||||
func seedRootAndData(root *string, link string) func(string) error {
|
||||
return func(target string) error {
|
||||
*root = target
|
||||
if err := os.WriteFile(filepath.Join(target, "f"), nil, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(target, "data"), 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Symlink(link, filepath.Join(target, "data", "f"))
|
||||
}
|
||||
}
|
||||
|
||||
func assertNotMigrated(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(filepath.Join(root, "f")); err != nil {
|
||||
t.Errorf("source was moved: %v", err)
|
||||
}
|
||||
if fi, err := os.Lstat(filepath.Join(root, "data", "f")); err != nil || fi.Mode()&os.ModeSymlink == 0 {
|
||||
t.Errorf("destination was replaced: %v, %v", fi, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A soft temp mount answers a stat with EIO while the server is unreachable;
|
||||
// a rename issued once it is back would replace the file the stat missed.
|
||||
func TestNodePublishVolumeNFSDoesNotMigrateOverAnUncheckedDestination(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
var root string
|
||||
w := &WrappedNodeService{
|
||||
mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")},
|
||||
lstat: func(string) (os.FileInfo, error) { return nil, syscall.EIO },
|
||||
}
|
||||
|
||||
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
|
||||
if status.Code(err) != codes.Internal {
|
||||
t.Fatalf("err = %v, want code Internal from the failed stat", err)
|
||||
}
|
||||
assertNotMigrated(t, root)
|
||||
}
|
||||
|
||||
// The symlinks are the user's: one that loops or dangles is still an entry
|
||||
// in /data, neither a reason to fail the publish nor one to replace it.
|
||||
func TestNodePublishVolumeNFSLeavesASymlinkInDataAlone(t *testing.T) {
|
||||
for name, link := range map[string]string{"loop": "f", "dangling": "missing"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
var root string
|
||||
w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, link)}}
|
||||
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
|
||||
t.Fatalf("NodePublishVolume: %v", err)
|
||||
}
|
||||
assertNotMigrated(t, root)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The root entry stays where the pod cannot see it, so the skip has to be
|
||||
// findable by an operator.
|
||||
func TestNodePublishVolumeNFSWarnsAboutAnEntryLeftInTheRoot(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
var logs bytes.Buffer
|
||||
klog.LogToStderr(false)
|
||||
klog.SetOutput(&logs)
|
||||
t.Cleanup(func() {
|
||||
klog.SetOutput(os.Stderr)
|
||||
klog.LogToStderr(true)
|
||||
})
|
||||
var root string
|
||||
w := &WrappedNodeService{mounter: &seedingMounter{FakeMounter: mount.NewFakeMounter(nil), seed: seedRootAndData(&root, "elsewhere")}}
|
||||
if _, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target"))); err != nil {
|
||||
t.Fatalf("NodePublishVolume: %v", err)
|
||||
}
|
||||
klog.Flush()
|
||||
for _, line := range strings.Split(logs.String(), "\n") {
|
||||
if strings.HasPrefix(line, "W") && strings.Contains(line, "volume pvc-1") && strings.Contains(line, "/data/f") {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("the skipped migration of f was not logged as a warning; log output:\n%s", logs.String())
|
||||
}
|
||||
|
||||
func TestNodePublishVolumeNFSTempMountErrorsNameTheVolume(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
tmpdir func(t *testing.T) string
|
||||
w *WrappedNodeService
|
||||
}{
|
||||
"temp dir": {
|
||||
tmpdir: func(t *testing.T) string { return filepath.Join(t.TempDir(), "missing") },
|
||||
w: &WrappedNodeService{mounter: mount.NewFakeMounter(nil)},
|
||||
},
|
||||
"mount": {
|
||||
tmpdir: func(t *testing.T) string { return t.TempDir() },
|
||||
w: &WrappedNodeService{mounter: failingMounter{mount.NewFakeMounter(nil)}},
|
||||
},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Setenv("TMPDIR", tc.tmpdir(t))
|
||||
_, err := tc.w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
|
||||
if status.Code(err) != codes.Internal || !strings.Contains(err.Error(), "volume pvc-1") {
|
||||
t.Errorf("err = %v, want code Internal naming volume pvc-1", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// On a dead server the stat inside MkdirAll fails on the temp mount point
|
||||
// itself, and MkdirAll then reports the local temp dir as already existing
|
||||
// instead of the errno the server's absence produced.
|
||||
func TestNodePublishVolumeNFSReportsTheDataDirErrno(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
w := &WrappedNodeService{
|
||||
mounter: mount.NewFakeMounter(nil),
|
||||
mkdir: func(string, os.FileMode) error { return syscall.EIO },
|
||||
}
|
||||
_, err := w.NodePublishVolume(context.Background(), nfsPublishRequest(filepath.Join(t.TempDir(), "target")))
|
||||
if err == nil || !strings.Contains(err.Error(), syscall.EIO.Error()) || !strings.Contains(err.Error(), "volume pvc-1") {
|
||||
t.Errorf("err = %v, want the EIO from mkdir, naming volume pvc-1", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user