wgengine/magicsock: rebind on bad socket errors

On macOS and iOS, UDP reads have been observed to fail with ENOTSOCK and
EBADF. Both errors indicate that the fd is no longer usable. The receive
function would retry reading from the bad socket until some other event,
like a link change, caused a rebind.

magicsock now rebinds when a read fails with EBADF or ENOTSOCK. Read and
write failures share a single 5s throttle to handle bursts of failures.

Updates #22097

Change-Id: Idd19f7094d9fc11e0f61a3207eb20281acfd5255
Signed-off-by: Nick Beirne <nickb@tailscale.com>
This commit is contained in:
Nick Beirne authored and Nick committed 2026-11-17 09:09:04 -05:00
1 parent 46ad4538b7
commit 54c4153604
10 files changed
+229 -19

No files matched your search

+5
View File
@@ -12,6 +12,11 @@ import (
"strings"
)
// IsBadSocketError always returns false on Plan 9.
func IsBadSocketError(err error) bool {
return false
}
// Reports whether err resulted from reading or writing to a closed or broken pipe.
func IsClosedPipeError(err error) bool {
// Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe
+6
View File
@@ -12,6 +12,12 @@ import (
"syscall"
)
// IsBadSocketError reports whether err indicates that the socket itself is no
// longer valid.
func IsBadSocketError(err error) bool {
return errors.Is(err, syscall.EBADF) || errors.Is(err, syscall.ENOTSOCK)
}
// Reports whether err resulted from reading or writing to a closed or broken pipe.
func IsClosedPipeError(err error) bool {
// EPIPE/ENOTCONN are common errors when a send fails due to a closed
+39
View File
@@ -0,0 +1,39 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build unix
package neterror
import (
"errors"
"net"
"os"
"syscall"
"testing"
)
func TestIsBadSocketError(t *testing.T) {
for _, tt := range []struct {
err error
want bool
}{
{nil, false},
{errors.New("unknown error"), false},
{syscall.EBADF, true},
{syscall.ENOTSOCK, true},
{syscall.EPIPE, false},
{syscall.EHOSTUNREACH, false},
{net.ErrClosed, false},
} {
if got := IsBadSocketError(tt.err); got != tt.want {
t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want)
}
wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{
Syscall: "recvfrom", Err: tt.err,
}}
if got := IsBadSocketError(wrapped); got != tt.want {
t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want)
}
}
}
+5
View File
@@ -11,6 +11,11 @@ import (
"io/fs"
)
// IsBadSocketError always returns false on this platform.
func IsBadSocketError(err error) bool {
return false
}
// Reports whether err resulted from reading or writing to a closed or broken pipe.
func IsClosedPipeError(err error) bool {
// Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe
+8
View File
@@ -29,3 +29,11 @@ func TestIsClosedPipeError(t *testing.T) {
}
}
}
func TestIsBadSocketError(t *testing.T) {
for _, err := range []error{nil, errors.New("unknown error"), fs.ErrClosed} {
if IsBadSocketError(err) {
t.Errorf("IsBadSocketError(%v) = true, want false", err)
}
}
}
+6
View File
@@ -12,6 +12,12 @@ import (
"golang.org/x/sys/windows"
)
// IsBadSocketError reports whether err indicates that the socket itself is no
// longer valid.
func IsBadSocketError(err error) bool {
return errors.Is(err, windows.WSAEBADF) || errors.Is(err, windows.WSAENOTSOCK)
}
// IsClosedPipeError reports whether err resulted from reading or writing to a closed or broken pipe.
func IsClosedPipeError(err error) bool {
return errors.Is(err, windows.ERROR_NO_DATA) ||
+28
View File
@@ -8,6 +8,8 @@ import (
"fmt"
"io"
"io/fs"
"net"
"os"
"syscall"
"testing"
@@ -36,3 +38,29 @@ func TestIsClosedPipeError(t *testing.T) {
}
}
}
func TestIsBadSocketError(t *testing.T) {
for _, tt := range []struct {
err error
want bool
}{
{nil, false},
{errors.New("unknown error"), false},
{windows.WSAEBADF, true},
{windows.WSAENOTSOCK, true},
{windows.WSAECONNRESET, false},
{syscall.EBADF, false},
{syscall.ENOTSOCK, false},
{net.ErrClosed, false},
} {
if got := IsBadSocketError(tt.err); got != tt.want {
t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want)
}
wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{
Syscall: "WSARecvFrom", Err: tt.err,
}}
if got := IsBadSocketError(wrapped); got != tt.want {
t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want)
}
}
}
+40 -13
View File
@@ -397,7 +397,8 @@ type Conn struct {
getPeerByKey func(key.NodePublic) (_ wgint.Peer, ok bool)
// lastErrRebind tracks the last time a rebind was performed after
// experiencing a write error, and is used to throttle the rate of rebinds.
// experiencing a read or write error, and is used to throttle the rate of
// rebinds.
lastErrRebind syncs.AtomicValue[time.Time]
// appliedDisableUDPGRO and appliedDisableUDPGSO cache the last UDP offload
@@ -1591,7 +1592,7 @@ func (c *Conn) sendUDPBatch(flow int, addr epAddr, buffs [][]byte, offset int) (
c.logf("magicsock: %s", errGSO.Error())
err = errGSO.RetryErr
} else {
c.maybeRebindOnError(err)
c.maybeRebindOnWriteError(err)
}
}
return err == nil, err
@@ -1610,7 +1611,7 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap
} else {
metricSendUDPError.Add(1)
}
c.maybeRebindOnError(err)
c.maybeRebindOnWriteError(err)
} else {
if sent && !isDisco {
switch {
@@ -1636,24 +1637,42 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap
return
}
// maybeRebindOnError performs a rebind and restun if the error is one that is
// known to be healed by a rebind, and the rebind is not throttled.
func (c *Conn) maybeRebindOnError(err error) {
// maybeRebindOnWriteError performs a rebind and restun if the error is one that is
// known to be healed by a rebind, and the rebind is not currently throttled.
func (c *Conn) maybeRebindOnWriteError(err error) {
ok, reason := shouldRebind(err)
if !ok {
return
}
c.maybeRebind(reason)
}
if c.lastErrRebind.Load().Before(time.Now().Add(-5 * time.Second)) {
c.logf("magicsock: performing rebind due to %q", reason)
c.lastErrRebind.Store(time.Now())
c.Rebind()
go c.ReSTUN(reason)
} else {
c.logf("magicsock: not performing %q rebind due to throttle", reason)
// maybeRebindOnReadError rebinds and restuns if err indicates the socket is no
// longer valid and the rebind is not currently throttled.
func (c *Conn) maybeRebindOnReadError(err error) {
if neterror.IsBadSocketError(err) {
c.maybeRebind("bad-socket")
}
}
// maybeRebind rebinds and restuns unless c is closing or another error-driven
// rebind happened within the last errRebindInterval.
func (c *Conn) maybeRebind(reason string) {
if c.closing.Load() {
c.logf("magicsock: not performing %q rebind, conn is closing", reason)
return
}
last := c.lastErrRebind.Load()
now := time.Now()
if now.Sub(last) < errRebindInterval || !c.lastErrRebind.CompareAndSwap(last, now) {
c.logf("magicsock: not performing %q rebind due to throttle", reason)
return
}
c.logf("magicsock: performing rebind due to %q", reason)
c.Rebind()
go c.ReSTUN(reason)
}
// sendUDPNetcheck sends b via UDP to addr. It is used exclusively by netcheck.
// It returns the number of bytes sent along with any error encountered. It
// returns errors.ErrUnsupported if the client is explicitly configured to only
@@ -1759,6 +1778,9 @@ type udpReceiveConfig struct {
}
const (
// errRebindInterval is the minimum time between error-driven rebinds.
errRebindInterval = 5 * time.Second
// readErrRetryDelay paces a [conn.ReceiveFunc]'s read errors.
readErrRetryDelay = 100 * time.Millisecond
@@ -1817,6 +1839,7 @@ func (c *Conn) mkReceiveFunc(cfg udpReceiveConfig) conn.ReceiveFunc {
healthItem.SetBroken(true)
}
c.logf("magicsock: %s: read error: %v", name, err)
c.maybeRebindOnReadError(err)
select {
case <-c.connCtx.Done():
@@ -3917,6 +3940,10 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur
ruc.mu.Lock()
defer ruc.mu.Unlock()
if c.closing.Load() {
return net.ErrClosed
}
if !hasUDPTransport {
ruc.setConnLocked(nil, c.udpSockets, "", c.controlKnobs)
return nil
+92 -4
View File
@@ -3587,7 +3587,7 @@ func TestShouldRebind(t *testing.T) {
}
}
func TestMaybeRebindOnError(t *testing.T) {
func TestMaybeRebindOnWriteError(t *testing.T) {
tstest.PanicOnLog()
tstest.ResourceCheck(t)
@@ -3606,7 +3606,7 @@ func TestMaybeRebindOnError(t *testing.T) {
defer conn.Close()
before := metricRebindCalls.Value()
conn.maybeRebindOnError(rebindErr)
conn.maybeRebindOnWriteError(rebindErr)
after := metricRebindCalls.Value()
if before+1 != after {
t.Errorf("should rebind on %#v", rebindErr)
@@ -3623,7 +3623,7 @@ func TestMaybeRebindOnError(t *testing.T) {
lastRebindTime := time.Now().Add(-1 * time.Second)
conn.lastErrRebind.Store(lastRebindTime)
before := metricRebindCalls.Value()
conn.maybeRebindOnError(err)
conn.maybeRebindOnWriteError(err)
after := metricRebindCalls.Value()
if before != after {
t.Errorf("should not rebind within 5 seconds of last")
@@ -3633,7 +3633,7 @@ func TestMaybeRebindOnError(t *testing.T) {
// rebind time.
time.Sleep(6 * time.Second)
conn.maybeRebindOnError(err)
conn.maybeRebindOnWriteError(err)
newTime := conn.lastErrRebind.Load()
if newTime == lastRebindTime {
t.Errorf("expected a rebind to occur")
@@ -3646,6 +3646,94 @@ func TestMaybeRebindOnError(t *testing.T) {
})
}
func TestMaybeRebindOnReadError(t *testing.T) {
tstest.PanicOnLog()
tstest.ResourceCheck(t)
for _, tt := range []struct {
errno error
rebind bool
}{
{syscall.EBADF, runtime.GOOS != "windows"},
{syscall.ENOTSOCK, runtime.GOOS != "windows"},
{syscall.Errno(10009), runtime.GOOS == "windows"},
{syscall.Errno(10038), runtime.GOOS == "windows"},
{syscall.ECONNRESET, false},
} {
t.Run(tt.errno.Error(), func(t *testing.T) {
c := newTestConn(t)
defer c.Close()
before := metricRebindCalls.Value()
c.maybeRebindOnReadError(readErr(tt.errno))
var want int64
if tt.rebind {
want = 1
}
if got := metricRebindCalls.Value() - before; got != want {
t.Errorf("rebinds = %d, want %d", got, want)
}
})
}
}
func TestMaybeRebind(t *testing.T) {
tstest.PanicOnLog()
tstest.ResourceCheck(t)
t.Run("closing", func(t *testing.T) {
c := newTestConn(t)
defer c.Close()
c.closing.Store(true)
before := metricRebindCalls.Value()
c.maybeRebind("test")
if got := metricRebindCalls.Value() - before; got != 0 {
t.Errorf("rebinds = %d, want 0", got)
}
if !c.lastErrRebind.Load().IsZero() {
t.Error("lastErrRebind updated while closing")
}
})
t.Run("concurrent", func(t *testing.T) {
c := newTestConn(t)
defer c.Close()
before := metricRebindCalls.Value()
const n = 10
var wg sync.WaitGroup
start := make(chan struct{})
for range n {
wg.Go(func() {
<-start
c.maybeRebind("test")
})
}
close(start)
wg.Wait()
if got := metricRebindCalls.Value() - before; got != 1 {
t.Errorf("%d concurrent maybeRebind calls rebound %d times, want 1", n, got)
}
})
}
func TestBindSocketAfterClose(t *testing.T) {
c := newTestConn(t)
c.Close()
for _, tt := range []struct {
ruc *RebindingUDPConn
network string
}{
{&c.pconn4, "udp4"},
{&c.pconn6, "udp6"},
} {
if err := c.bindSocket(tt.ruc, tt.network, keepCurrentPort); !errors.Is(err, net.ErrClosed) {
t.Errorf("bindSocket(%s) after Close = %v, want net.ErrClosed", tt.network, err)
}
if port := tt.ruc.Port(); port != 0 {
t.Errorf("%s bound to port %d after Close, want unbound", tt.network, port)
}
}
}
func newTestConnAndRegistry(t *testing.T, mod ...func(*Options)) (*Conn, *usermetric.Registry) {
t.Helper()
bus := eventbus.New()
-2
View File
@@ -53,8 +53,6 @@ func TestReceiveFuncRetriesErrors(t *testing.T) {
syscall.Errno(10051),
syscall.EADDRNOTAVAIL,
syscall.Errno(10049),
syscall.EBADF,
syscall.ENOTSOCK,
syscall.ECONNRESET,
syscall.ECONNREFUSED,
errors.New("unexpected error"),