From 54c4153604e9f058ac50e77ab5c4e3ffc33ec697 Mon Sep 17 00:00:00 2001 From: Nick Beirne Date: Thu, 5 Nov 2026 17:57:45 +0000 Subject: [PATCH] wgengine/magicsock: rebind on bad socket errors On macOS and iOS, UDP reads have been observed to fail with ENOTSOCK and EBADF. Both errors indicate that the fd is no longer usable. The receive function would retry reading from the bad socket until some other event, like a link change, caused a rebind. magicsock now rebinds when a read fails with EBADF or ENOTSOCK. Read and write failures share a single 5s throttle to handle bursts of failures. Updates #22097 Change-Id: Idd19f7094d9fc11e0f61a3207eb20281acfd5255 Signed-off-by: Nick Beirne --- net/neterror/neterror_plan9.go | 5 ++ net/neterror/neterror_unix.go | 6 ++ net/neterror/neterror_unix_test.go | 39 ++++++++++ net/neterror/neterror_wasm.go | 5 ++ net/neterror/neterror_wasm_test.go | 8 ++ net/neterror/neterror_windows.go | 6 ++ net/neterror/neterror_windows_test.go | 28 +++++++ wgengine/magicsock/magicsock.go | 53 +++++++++---- wgengine/magicsock/magicsock_test.go | 96 +++++++++++++++++++++++- wgengine/magicsock/receive_abort_test.go | 2 - 10 files changed, 229 insertions(+), 19 deletions(-) create mode 100644 net/neterror/neterror_unix_test.go diff --git a/net/neterror/neterror_plan9.go b/net/neterror/neterror_plan9.go index a60c4dd64..1e5657677 100644 --- a/net/neterror/neterror_plan9.go +++ b/net/neterror/neterror_plan9.go @@ -12,6 +12,11 @@ import ( "strings" ) +// IsBadSocketError always returns false on Plan 9. +func IsBadSocketError(err error) bool { + return false +} + // Reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { // Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe diff --git a/net/neterror/neterror_unix.go b/net/neterror/neterror_unix.go index 9b999213c..00608bf33 100644 --- a/net/neterror/neterror_unix.go +++ b/net/neterror/neterror_unix.go @@ -12,6 +12,12 @@ import ( "syscall" ) +// IsBadSocketError reports whether err indicates that the socket itself is no +// longer valid. +func IsBadSocketError(err error) bool { + return errors.Is(err, syscall.EBADF) || errors.Is(err, syscall.ENOTSOCK) +} + // Reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { // EPIPE/ENOTCONN are common errors when a send fails due to a closed diff --git a/net/neterror/neterror_unix_test.go b/net/neterror/neterror_unix_test.go new file mode 100644 index 000000000..113247aec --- /dev/null +++ b/net/neterror/neterror_unix_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package neterror + +import ( + "errors" + "net" + "os" + "syscall" + "testing" +) + +func TestIsBadSocketError(t *testing.T) { + for _, tt := range []struct { + err error + want bool + }{ + {nil, false}, + {errors.New("unknown error"), false}, + {syscall.EBADF, true}, + {syscall.ENOTSOCK, true}, + {syscall.EPIPE, false}, + {syscall.EHOSTUNREACH, false}, + {net.ErrClosed, false}, + } { + if got := IsBadSocketError(tt.err); got != tt.want { + t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want) + } + wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{ + Syscall: "recvfrom", Err: tt.err, + }} + if got := IsBadSocketError(wrapped); got != tt.want { + t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want) + } + } +} diff --git a/net/neterror/neterror_wasm.go b/net/neterror/neterror_wasm.go index 591367120..d413d4791 100644 --- a/net/neterror/neterror_wasm.go +++ b/net/neterror/neterror_wasm.go @@ -11,6 +11,11 @@ import ( "io/fs" ) +// IsBadSocketError always returns false on this platform. +func IsBadSocketError(err error) bool { + return false +} + // Reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { // Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe diff --git a/net/neterror/neterror_wasm_test.go b/net/neterror/neterror_wasm_test.go index 0703609dc..9db88dec6 100644 --- a/net/neterror/neterror_wasm_test.go +++ b/net/neterror/neterror_wasm_test.go @@ -29,3 +29,11 @@ func TestIsClosedPipeError(t *testing.T) { } } } + +func TestIsBadSocketError(t *testing.T) { + for _, err := range []error{nil, errors.New("unknown error"), fs.ErrClosed} { + if IsBadSocketError(err) { + t.Errorf("IsBadSocketError(%v) = true, want false", err) + } + } +} diff --git a/net/neterror/neterror_windows.go b/net/neterror/neterror_windows.go index 1d2d2817a..a111cabef 100644 --- a/net/neterror/neterror_windows.go +++ b/net/neterror/neterror_windows.go @@ -12,6 +12,12 @@ import ( "golang.org/x/sys/windows" ) +// IsBadSocketError reports whether err indicates that the socket itself is no +// longer valid. +func IsBadSocketError(err error) bool { + return errors.Is(err, windows.WSAEBADF) || errors.Is(err, windows.WSAENOTSOCK) +} + // IsClosedPipeError reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { return errors.Is(err, windows.ERROR_NO_DATA) || diff --git a/net/neterror/neterror_windows_test.go b/net/neterror/neterror_windows_test.go index ae895b141..506ce5dd2 100644 --- a/net/neterror/neterror_windows_test.go +++ b/net/neterror/neterror_windows_test.go @@ -8,6 +8,8 @@ import ( "fmt" "io" "io/fs" + "net" + "os" "syscall" "testing" @@ -36,3 +38,29 @@ func TestIsClosedPipeError(t *testing.T) { } } } + +func TestIsBadSocketError(t *testing.T) { + for _, tt := range []struct { + err error + want bool + }{ + {nil, false}, + {errors.New("unknown error"), false}, + {windows.WSAEBADF, true}, + {windows.WSAENOTSOCK, true}, + {windows.WSAECONNRESET, false}, + {syscall.EBADF, false}, + {syscall.ENOTSOCK, false}, + {net.ErrClosed, false}, + } { + if got := IsBadSocketError(tt.err); got != tt.want { + t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want) + } + wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{ + Syscall: "WSARecvFrom", Err: tt.err, + }} + if got := IsBadSocketError(wrapped); got != tt.want { + t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want) + } + } +} diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 4189a2884..faadeda39 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -397,7 +397,8 @@ type Conn struct { getPeerByKey func(key.NodePublic) (_ wgint.Peer, ok bool) // lastErrRebind tracks the last time a rebind was performed after - // experiencing a write error, and is used to throttle the rate of rebinds. + // experiencing a read or write error, and is used to throttle the rate of + // rebinds. lastErrRebind syncs.AtomicValue[time.Time] // appliedDisableUDPGRO and appliedDisableUDPGSO cache the last UDP offload @@ -1591,7 +1592,7 @@ func (c *Conn) sendUDPBatch(flow int, addr epAddr, buffs [][]byte, offset int) ( c.logf("magicsock: %s", errGSO.Error()) err = errGSO.RetryErr } else { - c.maybeRebindOnError(err) + c.maybeRebindOnWriteError(err) } } return err == nil, err @@ -1610,7 +1611,7 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap } else { metricSendUDPError.Add(1) } - c.maybeRebindOnError(err) + c.maybeRebindOnWriteError(err) } else { if sent && !isDisco { switch { @@ -1636,24 +1637,42 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap return } -// maybeRebindOnError performs a rebind and restun if the error is one that is -// known to be healed by a rebind, and the rebind is not throttled. -func (c *Conn) maybeRebindOnError(err error) { +// maybeRebindOnWriteError performs a rebind and restun if the error is one that is +// known to be healed by a rebind, and the rebind is not currently throttled. +func (c *Conn) maybeRebindOnWriteError(err error) { ok, reason := shouldRebind(err) if !ok { return } + c.maybeRebind(reason) +} - if c.lastErrRebind.Load().Before(time.Now().Add(-5 * time.Second)) { - c.logf("magicsock: performing rebind due to %q", reason) - c.lastErrRebind.Store(time.Now()) - c.Rebind() - go c.ReSTUN(reason) - } else { - c.logf("magicsock: not performing %q rebind due to throttle", reason) +// maybeRebindOnReadError rebinds and restuns if err indicates the socket is no +// longer valid and the rebind is not currently throttled. +func (c *Conn) maybeRebindOnReadError(err error) { + if neterror.IsBadSocketError(err) { + c.maybeRebind("bad-socket") } } +// maybeRebind rebinds and restuns unless c is closing or another error-driven +// rebind happened within the last errRebindInterval. +func (c *Conn) maybeRebind(reason string) { + if c.closing.Load() { + c.logf("magicsock: not performing %q rebind, conn is closing", reason) + return + } + last := c.lastErrRebind.Load() + now := time.Now() + if now.Sub(last) < errRebindInterval || !c.lastErrRebind.CompareAndSwap(last, now) { + c.logf("magicsock: not performing %q rebind due to throttle", reason) + return + } + c.logf("magicsock: performing rebind due to %q", reason) + c.Rebind() + go c.ReSTUN(reason) +} + // sendUDPNetcheck sends b via UDP to addr. It is used exclusively by netcheck. // It returns the number of bytes sent along with any error encountered. It // returns errors.ErrUnsupported if the client is explicitly configured to only @@ -1759,6 +1778,9 @@ type udpReceiveConfig struct { } const ( + // errRebindInterval is the minimum time between error-driven rebinds. + errRebindInterval = 5 * time.Second + // readErrRetryDelay paces a [conn.ReceiveFunc]'s read errors. readErrRetryDelay = 100 * time.Millisecond @@ -1817,6 +1839,7 @@ func (c *Conn) mkReceiveFunc(cfg udpReceiveConfig) conn.ReceiveFunc { healthItem.SetBroken(true) } c.logf("magicsock: %s: read error: %v", name, err) + c.maybeRebindOnReadError(err) select { case <-c.connCtx.Done(): @@ -3917,6 +3940,10 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur ruc.mu.Lock() defer ruc.mu.Unlock() + if c.closing.Load() { + return net.ErrClosed + } + if !hasUDPTransport { ruc.setConnLocked(nil, c.udpSockets, "", c.controlKnobs) return nil diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 3cb2c5f5d..a17f14c74 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3587,7 +3587,7 @@ func TestShouldRebind(t *testing.T) { } } -func TestMaybeRebindOnError(t *testing.T) { +func TestMaybeRebindOnWriteError(t *testing.T) { tstest.PanicOnLog() tstest.ResourceCheck(t) @@ -3606,7 +3606,7 @@ func TestMaybeRebindOnError(t *testing.T) { defer conn.Close() before := metricRebindCalls.Value() - conn.maybeRebindOnError(rebindErr) + conn.maybeRebindOnWriteError(rebindErr) after := metricRebindCalls.Value() if before+1 != after { t.Errorf("should rebind on %#v", rebindErr) @@ -3623,7 +3623,7 @@ func TestMaybeRebindOnError(t *testing.T) { lastRebindTime := time.Now().Add(-1 * time.Second) conn.lastErrRebind.Store(lastRebindTime) before := metricRebindCalls.Value() - conn.maybeRebindOnError(err) + conn.maybeRebindOnWriteError(err) after := metricRebindCalls.Value() if before != after { t.Errorf("should not rebind within 5 seconds of last") @@ -3633,7 +3633,7 @@ func TestMaybeRebindOnError(t *testing.T) { // rebind time. time.Sleep(6 * time.Second) - conn.maybeRebindOnError(err) + conn.maybeRebindOnWriteError(err) newTime := conn.lastErrRebind.Load() if newTime == lastRebindTime { t.Errorf("expected a rebind to occur") @@ -3646,6 +3646,94 @@ func TestMaybeRebindOnError(t *testing.T) { }) } +func TestMaybeRebindOnReadError(t *testing.T) { + tstest.PanicOnLog() + tstest.ResourceCheck(t) + + for _, tt := range []struct { + errno error + rebind bool + }{ + {syscall.EBADF, runtime.GOOS != "windows"}, + {syscall.ENOTSOCK, runtime.GOOS != "windows"}, + {syscall.Errno(10009), runtime.GOOS == "windows"}, + {syscall.Errno(10038), runtime.GOOS == "windows"}, + {syscall.ECONNRESET, false}, + } { + t.Run(tt.errno.Error(), func(t *testing.T) { + c := newTestConn(t) + defer c.Close() + before := metricRebindCalls.Value() + c.maybeRebindOnReadError(readErr(tt.errno)) + var want int64 + if tt.rebind { + want = 1 + } + if got := metricRebindCalls.Value() - before; got != want { + t.Errorf("rebinds = %d, want %d", got, want) + } + }) + } +} + +func TestMaybeRebind(t *testing.T) { + tstest.PanicOnLog() + tstest.ResourceCheck(t) + + t.Run("closing", func(t *testing.T) { + c := newTestConn(t) + defer c.Close() + c.closing.Store(true) + before := metricRebindCalls.Value() + c.maybeRebind("test") + if got := metricRebindCalls.Value() - before; got != 0 { + t.Errorf("rebinds = %d, want 0", got) + } + if !c.lastErrRebind.Load().IsZero() { + t.Error("lastErrRebind updated while closing") + } + }) + + t.Run("concurrent", func(t *testing.T) { + c := newTestConn(t) + defer c.Close() + before := metricRebindCalls.Value() + const n = 10 + var wg sync.WaitGroup + start := make(chan struct{}) + for range n { + wg.Go(func() { + <-start + c.maybeRebind("test") + }) + } + close(start) + wg.Wait() + if got := metricRebindCalls.Value() - before; got != 1 { + t.Errorf("%d concurrent maybeRebind calls rebound %d times, want 1", n, got) + } + }) +} + +func TestBindSocketAfterClose(t *testing.T) { + c := newTestConn(t) + c.Close() + for _, tt := range []struct { + ruc *RebindingUDPConn + network string + }{ + {&c.pconn4, "udp4"}, + {&c.pconn6, "udp6"}, + } { + if err := c.bindSocket(tt.ruc, tt.network, keepCurrentPort); !errors.Is(err, net.ErrClosed) { + t.Errorf("bindSocket(%s) after Close = %v, want net.ErrClosed", tt.network, err) + } + if port := tt.ruc.Port(); port != 0 { + t.Errorf("%s bound to port %d after Close, want unbound", tt.network, port) + } + } +} + func newTestConnAndRegistry(t *testing.T, mod ...func(*Options)) (*Conn, *usermetric.Registry) { t.Helper() bus := eventbus.New() diff --git a/wgengine/magicsock/receive_abort_test.go b/wgengine/magicsock/receive_abort_test.go index 8af66878d..3d3e2ec09 100644 --- a/wgengine/magicsock/receive_abort_test.go +++ b/wgengine/magicsock/receive_abort_test.go @@ -53,8 +53,6 @@ func TestReceiveFuncRetriesErrors(t *testing.T) { syscall.Errno(10051), syscall.EADDRNOTAVAIL, syscall.Errno(10049), - syscall.EBADF, - syscall.ENOTSOCK, syscall.ECONNRESET, syscall.ECONNREFUSED, errors.New("unexpected error"),