diff --git a/net/neterror/neterror_plan9.go b/net/neterror/neterror_plan9.go index a60c4dd64..1e5657677 100644 --- a/net/neterror/neterror_plan9.go +++ b/net/neterror/neterror_plan9.go @@ -12,6 +12,11 @@ import ( "strings" ) +// IsBadSocketError always returns false on Plan 9. +func IsBadSocketError(err error) bool { + return false +} + // Reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { // Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe diff --git a/net/neterror/neterror_unix.go b/net/neterror/neterror_unix.go index 9b999213c..00608bf33 100644 --- a/net/neterror/neterror_unix.go +++ b/net/neterror/neterror_unix.go @@ -12,6 +12,12 @@ import ( "syscall" ) +// IsBadSocketError reports whether err indicates that the socket itself is no +// longer valid. +func IsBadSocketError(err error) bool { + return errors.Is(err, syscall.EBADF) || errors.Is(err, syscall.ENOTSOCK) +} + // Reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { // EPIPE/ENOTCONN are common errors when a send fails due to a closed diff --git a/net/neterror/neterror_unix_test.go b/net/neterror/neterror_unix_test.go new file mode 100644 index 000000000..113247aec --- /dev/null +++ b/net/neterror/neterror_unix_test.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package neterror + +import ( + "errors" + "net" + "os" + "syscall" + "testing" +) + +func TestIsBadSocketError(t *testing.T) { + for _, tt := range []struct { + err error + want bool + }{ + {nil, false}, + {errors.New("unknown error"), false}, + {syscall.EBADF, true}, + {syscall.ENOTSOCK, true}, + {syscall.EPIPE, false}, + {syscall.EHOSTUNREACH, false}, + {net.ErrClosed, false}, + } { + if got := IsBadSocketError(tt.err); got != tt.want { + t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want) + } + wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{ + Syscall: "recvfrom", Err: tt.err, + }} + if got := IsBadSocketError(wrapped); got != tt.want { + t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want) + } + } +} diff --git a/net/neterror/neterror_wasm.go b/net/neterror/neterror_wasm.go index 591367120..d413d4791 100644 --- a/net/neterror/neterror_wasm.go +++ b/net/neterror/neterror_wasm.go @@ -11,6 +11,11 @@ import ( "io/fs" ) +// IsBadSocketError always returns false on this platform. +func IsBadSocketError(err error) bool { + return false +} + // Reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { // Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe diff --git a/net/neterror/neterror_wasm_test.go b/net/neterror/neterror_wasm_test.go index 0703609dc..9db88dec6 100644 --- a/net/neterror/neterror_wasm_test.go +++ b/net/neterror/neterror_wasm_test.go @@ -29,3 +29,11 @@ func TestIsClosedPipeError(t *testing.T) { } } } + +func TestIsBadSocketError(t *testing.T) { + for _, err := range []error{nil, errors.New("unknown error"), fs.ErrClosed} { + if IsBadSocketError(err) { + t.Errorf("IsBadSocketError(%v) = true, want false", err) + } + } +} diff --git a/net/neterror/neterror_windows.go b/net/neterror/neterror_windows.go index 1d2d2817a..a111cabef 100644 --- a/net/neterror/neterror_windows.go +++ b/net/neterror/neterror_windows.go @@ -12,6 +12,12 @@ import ( "golang.org/x/sys/windows" ) +// IsBadSocketError reports whether err indicates that the socket itself is no +// longer valid. +func IsBadSocketError(err error) bool { + return errors.Is(err, windows.WSAEBADF) || errors.Is(err, windows.WSAENOTSOCK) +} + // IsClosedPipeError reports whether err resulted from reading or writing to a closed or broken pipe. func IsClosedPipeError(err error) bool { return errors.Is(err, windows.ERROR_NO_DATA) || diff --git a/net/neterror/neterror_windows_test.go b/net/neterror/neterror_windows_test.go index ae895b141..506ce5dd2 100644 --- a/net/neterror/neterror_windows_test.go +++ b/net/neterror/neterror_windows_test.go @@ -8,6 +8,8 @@ import ( "fmt" "io" "io/fs" + "net" + "os" "syscall" "testing" @@ -36,3 +38,29 @@ func TestIsClosedPipeError(t *testing.T) { } } } + +func TestIsBadSocketError(t *testing.T) { + for _, tt := range []struct { + err error + want bool + }{ + {nil, false}, + {errors.New("unknown error"), false}, + {windows.WSAEBADF, true}, + {windows.WSAENOTSOCK, true}, + {windows.WSAECONNRESET, false}, + {syscall.EBADF, false}, + {syscall.ENOTSOCK, false}, + {net.ErrClosed, false}, + } { + if got := IsBadSocketError(tt.err); got != tt.want { + t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want) + } + wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{ + Syscall: "WSARecvFrom", Err: tt.err, + }} + if got := IsBadSocketError(wrapped); got != tt.want { + t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want) + } + } +} diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 4189a2884..faadeda39 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -397,7 +397,8 @@ type Conn struct { getPeerByKey func(key.NodePublic) (_ wgint.Peer, ok bool) // lastErrRebind tracks the last time a rebind was performed after - // experiencing a write error, and is used to throttle the rate of rebinds. + // experiencing a read or write error, and is used to throttle the rate of + // rebinds. lastErrRebind syncs.AtomicValue[time.Time] // appliedDisableUDPGRO and appliedDisableUDPGSO cache the last UDP offload @@ -1591,7 +1592,7 @@ func (c *Conn) sendUDPBatch(flow int, addr epAddr, buffs [][]byte, offset int) ( c.logf("magicsock: %s", errGSO.Error()) err = errGSO.RetryErr } else { - c.maybeRebindOnError(err) + c.maybeRebindOnWriteError(err) } } return err == nil, err @@ -1610,7 +1611,7 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap } else { metricSendUDPError.Add(1) } - c.maybeRebindOnError(err) + c.maybeRebindOnWriteError(err) } else { if sent && !isDisco { switch { @@ -1636,24 +1637,42 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap return } -// maybeRebindOnError performs a rebind and restun if the error is one that is -// known to be healed by a rebind, and the rebind is not throttled. -func (c *Conn) maybeRebindOnError(err error) { +// maybeRebindOnWriteError performs a rebind and restun if the error is one that is +// known to be healed by a rebind, and the rebind is not currently throttled. +func (c *Conn) maybeRebindOnWriteError(err error) { ok, reason := shouldRebind(err) if !ok { return } + c.maybeRebind(reason) +} - if c.lastErrRebind.Load().Before(time.Now().Add(-5 * time.Second)) { - c.logf("magicsock: performing rebind due to %q", reason) - c.lastErrRebind.Store(time.Now()) - c.Rebind() - go c.ReSTUN(reason) - } else { - c.logf("magicsock: not performing %q rebind due to throttle", reason) +// maybeRebindOnReadError rebinds and restuns if err indicates the socket is no +// longer valid and the rebind is not currently throttled. +func (c *Conn) maybeRebindOnReadError(err error) { + if neterror.IsBadSocketError(err) { + c.maybeRebind("bad-socket") } } +// maybeRebind rebinds and restuns unless c is closing or another error-driven +// rebind happened within the last errRebindInterval. +func (c *Conn) maybeRebind(reason string) { + if c.closing.Load() { + c.logf("magicsock: not performing %q rebind, conn is closing", reason) + return + } + last := c.lastErrRebind.Load() + now := time.Now() + if now.Sub(last) < errRebindInterval || !c.lastErrRebind.CompareAndSwap(last, now) { + c.logf("magicsock: not performing %q rebind due to throttle", reason) + return + } + c.logf("magicsock: performing rebind due to %q", reason) + c.Rebind() + go c.ReSTUN(reason) +} + // sendUDPNetcheck sends b via UDP to addr. It is used exclusively by netcheck. // It returns the number of bytes sent along with any error encountered. It // returns errors.ErrUnsupported if the client is explicitly configured to only @@ -1759,6 +1778,9 @@ type udpReceiveConfig struct { } const ( + // errRebindInterval is the minimum time between error-driven rebinds. + errRebindInterval = 5 * time.Second + // readErrRetryDelay paces a [conn.ReceiveFunc]'s read errors. readErrRetryDelay = 100 * time.Millisecond @@ -1817,6 +1839,7 @@ func (c *Conn) mkReceiveFunc(cfg udpReceiveConfig) conn.ReceiveFunc { healthItem.SetBroken(true) } c.logf("magicsock: %s: read error: %v", name, err) + c.maybeRebindOnReadError(err) select { case <-c.connCtx.Done(): @@ -3917,6 +3940,10 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur ruc.mu.Lock() defer ruc.mu.Unlock() + if c.closing.Load() { + return net.ErrClosed + } + if !hasUDPTransport { ruc.setConnLocked(nil, c.udpSockets, "", c.controlKnobs) return nil diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 3cb2c5f5d..a17f14c74 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3587,7 +3587,7 @@ func TestShouldRebind(t *testing.T) { } } -func TestMaybeRebindOnError(t *testing.T) { +func TestMaybeRebindOnWriteError(t *testing.T) { tstest.PanicOnLog() tstest.ResourceCheck(t) @@ -3606,7 +3606,7 @@ func TestMaybeRebindOnError(t *testing.T) { defer conn.Close() before := metricRebindCalls.Value() - conn.maybeRebindOnError(rebindErr) + conn.maybeRebindOnWriteError(rebindErr) after := metricRebindCalls.Value() if before+1 != after { t.Errorf("should rebind on %#v", rebindErr) @@ -3623,7 +3623,7 @@ func TestMaybeRebindOnError(t *testing.T) { lastRebindTime := time.Now().Add(-1 * time.Second) conn.lastErrRebind.Store(lastRebindTime) before := metricRebindCalls.Value() - conn.maybeRebindOnError(err) + conn.maybeRebindOnWriteError(err) after := metricRebindCalls.Value() if before != after { t.Errorf("should not rebind within 5 seconds of last") @@ -3633,7 +3633,7 @@ func TestMaybeRebindOnError(t *testing.T) { // rebind time. time.Sleep(6 * time.Second) - conn.maybeRebindOnError(err) + conn.maybeRebindOnWriteError(err) newTime := conn.lastErrRebind.Load() if newTime == lastRebindTime { t.Errorf("expected a rebind to occur") @@ -3646,6 +3646,94 @@ func TestMaybeRebindOnError(t *testing.T) { }) } +func TestMaybeRebindOnReadError(t *testing.T) { + tstest.PanicOnLog() + tstest.ResourceCheck(t) + + for _, tt := range []struct { + errno error + rebind bool + }{ + {syscall.EBADF, runtime.GOOS != "windows"}, + {syscall.ENOTSOCK, runtime.GOOS != "windows"}, + {syscall.Errno(10009), runtime.GOOS == "windows"}, + {syscall.Errno(10038), runtime.GOOS == "windows"}, + {syscall.ECONNRESET, false}, + } { + t.Run(tt.errno.Error(), func(t *testing.T) { + c := newTestConn(t) + defer c.Close() + before := metricRebindCalls.Value() + c.maybeRebindOnReadError(readErr(tt.errno)) + var want int64 + if tt.rebind { + want = 1 + } + if got := metricRebindCalls.Value() - before; got != want { + t.Errorf("rebinds = %d, want %d", got, want) + } + }) + } +} + +func TestMaybeRebind(t *testing.T) { + tstest.PanicOnLog() + tstest.ResourceCheck(t) + + t.Run("closing", func(t *testing.T) { + c := newTestConn(t) + defer c.Close() + c.closing.Store(true) + before := metricRebindCalls.Value() + c.maybeRebind("test") + if got := metricRebindCalls.Value() - before; got != 0 { + t.Errorf("rebinds = %d, want 0", got) + } + if !c.lastErrRebind.Load().IsZero() { + t.Error("lastErrRebind updated while closing") + } + }) + + t.Run("concurrent", func(t *testing.T) { + c := newTestConn(t) + defer c.Close() + before := metricRebindCalls.Value() + const n = 10 + var wg sync.WaitGroup + start := make(chan struct{}) + for range n { + wg.Go(func() { + <-start + c.maybeRebind("test") + }) + } + close(start) + wg.Wait() + if got := metricRebindCalls.Value() - before; got != 1 { + t.Errorf("%d concurrent maybeRebind calls rebound %d times, want 1", n, got) + } + }) +} + +func TestBindSocketAfterClose(t *testing.T) { + c := newTestConn(t) + c.Close() + for _, tt := range []struct { + ruc *RebindingUDPConn + network string + }{ + {&c.pconn4, "udp4"}, + {&c.pconn6, "udp6"}, + } { + if err := c.bindSocket(tt.ruc, tt.network, keepCurrentPort); !errors.Is(err, net.ErrClosed) { + t.Errorf("bindSocket(%s) after Close = %v, want net.ErrClosed", tt.network, err) + } + if port := tt.ruc.Port(); port != 0 { + t.Errorf("%s bound to port %d after Close, want unbound", tt.network, port) + } + } +} + func newTestConnAndRegistry(t *testing.T, mod ...func(*Options)) (*Conn, *usermetric.Registry) { t.Helper() bus := eventbus.New() diff --git a/wgengine/magicsock/receive_abort_test.go b/wgengine/magicsock/receive_abort_test.go index 8af66878d..3d3e2ec09 100644 --- a/wgengine/magicsock/receive_abort_test.go +++ b/wgengine/magicsock/receive_abort_test.go @@ -53,8 +53,6 @@ func TestReceiveFuncRetriesErrors(t *testing.T) { syscall.Errno(10051), syscall.EADDRNOTAVAIL, syscall.Errno(10049), - syscall.EBADF, - syscall.ENOTSOCK, syscall.ECONNRESET, syscall.ECONNREFUSED, errors.New("unexpected error"),