mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-21 20:22:41 +09:00
wgengine/magicsock: rebind on bad socket errors
On macOS and iOS, UDP reads have been observed to fail with ENOTSOCK and EBADF. Both errors indicate that the fd is no longer usable. The receive function would retry reading from the bad socket until some other event, like a link change, caused a rebind. magicsock now rebinds when a read fails with EBADF or ENOTSOCK. Read and write failures share a single 5s throttle to handle bursts of failures. Updates #22097 Change-Id: Idd19f7094d9fc11e0f61a3207eb20281acfd5255 Signed-off-by: Nick Beirne <nickb@tailscale.com>
This commit is contained in:
1 parent
46ad4538b7
commit
54c4153604
10 files changed
+229
-19
No files matched your search
@@ -12,6 +12,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// IsBadSocketError always returns false on Plan 9.
|
||||
func IsBadSocketError(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
// Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe
|
||||
|
||||
@@ -12,6 +12,12 @@ import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// IsBadSocketError reports whether err indicates that the socket itself is no
|
||||
// longer valid.
|
||||
func IsBadSocketError(err error) bool {
|
||||
return errors.Is(err, syscall.EBADF) || errors.Is(err, syscall.ENOTSOCK)
|
||||
}
|
||||
|
||||
// Reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
// EPIPE/ENOTCONN are common errors when a send fails due to a closed
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build unix
|
||||
|
||||
package neterror
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsBadSocketError(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{nil, false},
|
||||
{errors.New("unknown error"), false},
|
||||
{syscall.EBADF, true},
|
||||
{syscall.ENOTSOCK, true},
|
||||
{syscall.EPIPE, false},
|
||||
{syscall.EHOSTUNREACH, false},
|
||||
{net.ErrClosed, false},
|
||||
} {
|
||||
if got := IsBadSocketError(tt.err); got != tt.want {
|
||||
t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{
|
||||
Syscall: "recvfrom", Err: tt.err,
|
||||
}}
|
||||
if got := IsBadSocketError(wrapped); got != tt.want {
|
||||
t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,11 @@ import (
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// IsBadSocketError always returns false on this platform.
|
||||
func IsBadSocketError(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
// Libraries may also return root errors like fs.ErrClosed/io.ErrClosedPipe
|
||||
|
||||
@@ -29,3 +29,11 @@ func TestIsClosedPipeError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBadSocketError(t *testing.T) {
|
||||
for _, err := range []error{nil, errors.New("unknown error"), fs.ErrClosed} {
|
||||
if IsBadSocketError(err) {
|
||||
t.Errorf("IsBadSocketError(%v) = true, want false", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,12 @@ import (
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// IsBadSocketError reports whether err indicates that the socket itself is no
|
||||
// longer valid.
|
||||
func IsBadSocketError(err error) bool {
|
||||
return errors.Is(err, windows.WSAEBADF) || errors.Is(err, windows.WSAENOTSOCK)
|
||||
}
|
||||
|
||||
// IsClosedPipeError reports whether err resulted from reading or writing to a closed or broken pipe.
|
||||
func IsClosedPipeError(err error) bool {
|
||||
return errors.Is(err, windows.ERROR_NO_DATA) ||
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
@@ -36,3 +38,29 @@ func TestIsClosedPipeError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBadSocketError(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{nil, false},
|
||||
{errors.New("unknown error"), false},
|
||||
{windows.WSAEBADF, true},
|
||||
{windows.WSAENOTSOCK, true},
|
||||
{windows.WSAECONNRESET, false},
|
||||
{syscall.EBADF, false},
|
||||
{syscall.ENOTSOCK, false},
|
||||
{net.ErrClosed, false},
|
||||
} {
|
||||
if got := IsBadSocketError(tt.err); got != tt.want {
|
||||
t.Errorf("IsBadSocketError(%v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
wrapped := &net.OpError{Op: "read", Net: "udp4", Err: &os.SyscallError{
|
||||
Syscall: "WSARecvFrom", Err: tt.err,
|
||||
}}
|
||||
if got := IsBadSocketError(wrapped); got != tt.want {
|
||||
t.Errorf("IsBadSocketError(wrapped %v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -397,7 +397,8 @@ type Conn struct {
|
||||
getPeerByKey func(key.NodePublic) (_ wgint.Peer, ok bool)
|
||||
|
||||
// lastErrRebind tracks the last time a rebind was performed after
|
||||
// experiencing a write error, and is used to throttle the rate of rebinds.
|
||||
// experiencing a read or write error, and is used to throttle the rate of
|
||||
// rebinds.
|
||||
lastErrRebind syncs.AtomicValue[time.Time]
|
||||
|
||||
// appliedDisableUDPGRO and appliedDisableUDPGSO cache the last UDP offload
|
||||
@@ -1591,7 +1592,7 @@ func (c *Conn) sendUDPBatch(flow int, addr epAddr, buffs [][]byte, offset int) (
|
||||
c.logf("magicsock: %s", errGSO.Error())
|
||||
err = errGSO.RetryErr
|
||||
} else {
|
||||
c.maybeRebindOnError(err)
|
||||
c.maybeRebindOnWriteError(err)
|
||||
}
|
||||
}
|
||||
return err == nil, err
|
||||
@@ -1610,7 +1611,7 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap
|
||||
} else {
|
||||
metricSendUDPError.Add(1)
|
||||
}
|
||||
c.maybeRebindOnError(err)
|
||||
c.maybeRebindOnWriteError(err)
|
||||
} else {
|
||||
if sent && !isDisco {
|
||||
switch {
|
||||
@@ -1636,24 +1637,42 @@ func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap
|
||||
return
|
||||
}
|
||||
|
||||
// maybeRebindOnError performs a rebind and restun if the error is one that is
|
||||
// known to be healed by a rebind, and the rebind is not throttled.
|
||||
func (c *Conn) maybeRebindOnError(err error) {
|
||||
// maybeRebindOnWriteError performs a rebind and restun if the error is one that is
|
||||
// known to be healed by a rebind, and the rebind is not currently throttled.
|
||||
func (c *Conn) maybeRebindOnWriteError(err error) {
|
||||
ok, reason := shouldRebind(err)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
c.maybeRebind(reason)
|
||||
}
|
||||
|
||||
if c.lastErrRebind.Load().Before(time.Now().Add(-5 * time.Second)) {
|
||||
c.logf("magicsock: performing rebind due to %q", reason)
|
||||
c.lastErrRebind.Store(time.Now())
|
||||
c.Rebind()
|
||||
go c.ReSTUN(reason)
|
||||
} else {
|
||||
c.logf("magicsock: not performing %q rebind due to throttle", reason)
|
||||
// maybeRebindOnReadError rebinds and restuns if err indicates the socket is no
|
||||
// longer valid and the rebind is not currently throttled.
|
||||
func (c *Conn) maybeRebindOnReadError(err error) {
|
||||
if neterror.IsBadSocketError(err) {
|
||||
c.maybeRebind("bad-socket")
|
||||
}
|
||||
}
|
||||
|
||||
// maybeRebind rebinds and restuns unless c is closing or another error-driven
|
||||
// rebind happened within the last errRebindInterval.
|
||||
func (c *Conn) maybeRebind(reason string) {
|
||||
if c.closing.Load() {
|
||||
c.logf("magicsock: not performing %q rebind, conn is closing", reason)
|
||||
return
|
||||
}
|
||||
last := c.lastErrRebind.Load()
|
||||
now := time.Now()
|
||||
if now.Sub(last) < errRebindInterval || !c.lastErrRebind.CompareAndSwap(last, now) {
|
||||
c.logf("magicsock: not performing %q rebind due to throttle", reason)
|
||||
return
|
||||
}
|
||||
c.logf("magicsock: performing rebind due to %q", reason)
|
||||
c.Rebind()
|
||||
go c.ReSTUN(reason)
|
||||
}
|
||||
|
||||
// sendUDPNetcheck sends b via UDP to addr. It is used exclusively by netcheck.
|
||||
// It returns the number of bytes sent along with any error encountered. It
|
||||
// returns errors.ErrUnsupported if the client is explicitly configured to only
|
||||
@@ -1759,6 +1778,9 @@ type udpReceiveConfig struct {
|
||||
}
|
||||
|
||||
const (
|
||||
// errRebindInterval is the minimum time between error-driven rebinds.
|
||||
errRebindInterval = 5 * time.Second
|
||||
|
||||
// readErrRetryDelay paces a [conn.ReceiveFunc]'s read errors.
|
||||
readErrRetryDelay = 100 * time.Millisecond
|
||||
|
||||
@@ -1817,6 +1839,7 @@ func (c *Conn) mkReceiveFunc(cfg udpReceiveConfig) conn.ReceiveFunc {
|
||||
healthItem.SetBroken(true)
|
||||
}
|
||||
c.logf("magicsock: %s: read error: %v", name, err)
|
||||
c.maybeRebindOnReadError(err)
|
||||
|
||||
select {
|
||||
case <-c.connCtx.Done():
|
||||
@@ -3917,6 +3940,10 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur
|
||||
ruc.mu.Lock()
|
||||
defer ruc.mu.Unlock()
|
||||
|
||||
if c.closing.Load() {
|
||||
return net.ErrClosed
|
||||
}
|
||||
|
||||
if !hasUDPTransport {
|
||||
ruc.setConnLocked(nil, c.udpSockets, "", c.controlKnobs)
|
||||
return nil
|
||||
|
||||
@@ -3587,7 +3587,7 @@ func TestShouldRebind(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRebindOnError(t *testing.T) {
|
||||
func TestMaybeRebindOnWriteError(t *testing.T) {
|
||||
tstest.PanicOnLog()
|
||||
tstest.ResourceCheck(t)
|
||||
|
||||
@@ -3606,7 +3606,7 @@ func TestMaybeRebindOnError(t *testing.T) {
|
||||
defer conn.Close()
|
||||
|
||||
before := metricRebindCalls.Value()
|
||||
conn.maybeRebindOnError(rebindErr)
|
||||
conn.maybeRebindOnWriteError(rebindErr)
|
||||
after := metricRebindCalls.Value()
|
||||
if before+1 != after {
|
||||
t.Errorf("should rebind on %#v", rebindErr)
|
||||
@@ -3623,7 +3623,7 @@ func TestMaybeRebindOnError(t *testing.T) {
|
||||
lastRebindTime := time.Now().Add(-1 * time.Second)
|
||||
conn.lastErrRebind.Store(lastRebindTime)
|
||||
before := metricRebindCalls.Value()
|
||||
conn.maybeRebindOnError(err)
|
||||
conn.maybeRebindOnWriteError(err)
|
||||
after := metricRebindCalls.Value()
|
||||
if before != after {
|
||||
t.Errorf("should not rebind within 5 seconds of last")
|
||||
@@ -3633,7 +3633,7 @@ func TestMaybeRebindOnError(t *testing.T) {
|
||||
// rebind time.
|
||||
time.Sleep(6 * time.Second)
|
||||
|
||||
conn.maybeRebindOnError(err)
|
||||
conn.maybeRebindOnWriteError(err)
|
||||
newTime := conn.lastErrRebind.Load()
|
||||
if newTime == lastRebindTime {
|
||||
t.Errorf("expected a rebind to occur")
|
||||
@@ -3646,6 +3646,94 @@ func TestMaybeRebindOnError(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestMaybeRebindOnReadError(t *testing.T) {
|
||||
tstest.PanicOnLog()
|
||||
tstest.ResourceCheck(t)
|
||||
|
||||
for _, tt := range []struct {
|
||||
errno error
|
||||
rebind bool
|
||||
}{
|
||||
{syscall.EBADF, runtime.GOOS != "windows"},
|
||||
{syscall.ENOTSOCK, runtime.GOOS != "windows"},
|
||||
{syscall.Errno(10009), runtime.GOOS == "windows"},
|
||||
{syscall.Errno(10038), runtime.GOOS == "windows"},
|
||||
{syscall.ECONNRESET, false},
|
||||
} {
|
||||
t.Run(tt.errno.Error(), func(t *testing.T) {
|
||||
c := newTestConn(t)
|
||||
defer c.Close()
|
||||
before := metricRebindCalls.Value()
|
||||
c.maybeRebindOnReadError(readErr(tt.errno))
|
||||
var want int64
|
||||
if tt.rebind {
|
||||
want = 1
|
||||
}
|
||||
if got := metricRebindCalls.Value() - before; got != want {
|
||||
t.Errorf("rebinds = %d, want %d", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRebind(t *testing.T) {
|
||||
tstest.PanicOnLog()
|
||||
tstest.ResourceCheck(t)
|
||||
|
||||
t.Run("closing", func(t *testing.T) {
|
||||
c := newTestConn(t)
|
||||
defer c.Close()
|
||||
c.closing.Store(true)
|
||||
before := metricRebindCalls.Value()
|
||||
c.maybeRebind("test")
|
||||
if got := metricRebindCalls.Value() - before; got != 0 {
|
||||
t.Errorf("rebinds = %d, want 0", got)
|
||||
}
|
||||
if !c.lastErrRebind.Load().IsZero() {
|
||||
t.Error("lastErrRebind updated while closing")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent", func(t *testing.T) {
|
||||
c := newTestConn(t)
|
||||
defer c.Close()
|
||||
before := metricRebindCalls.Value()
|
||||
const n = 10
|
||||
var wg sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
for range n {
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
c.maybeRebind("test")
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if got := metricRebindCalls.Value() - before; got != 1 {
|
||||
t.Errorf("%d concurrent maybeRebind calls rebound %d times, want 1", n, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestBindSocketAfterClose(t *testing.T) {
|
||||
c := newTestConn(t)
|
||||
c.Close()
|
||||
for _, tt := range []struct {
|
||||
ruc *RebindingUDPConn
|
||||
network string
|
||||
}{
|
||||
{&c.pconn4, "udp4"},
|
||||
{&c.pconn6, "udp6"},
|
||||
} {
|
||||
if err := c.bindSocket(tt.ruc, tt.network, keepCurrentPort); !errors.Is(err, net.ErrClosed) {
|
||||
t.Errorf("bindSocket(%s) after Close = %v, want net.ErrClosed", tt.network, err)
|
||||
}
|
||||
if port := tt.ruc.Port(); port != 0 {
|
||||
t.Errorf("%s bound to port %d after Close, want unbound", tt.network, port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newTestConnAndRegistry(t *testing.T, mod ...func(*Options)) (*Conn, *usermetric.Registry) {
|
||||
t.Helper()
|
||||
bus := eventbus.New()
|
||||
|
||||
@@ -53,8 +53,6 @@ func TestReceiveFuncRetriesErrors(t *testing.T) {
|
||||
syscall.Errno(10051),
|
||||
syscall.EADDRNOTAVAIL,
|
||||
syscall.Errno(10049),
|
||||
syscall.EBADF,
|
||||
syscall.ENOTSOCK,
|
||||
syscall.ECONNRESET,
|
||||
syscall.ECONNREFUSED,
|
||||
errors.New("unexpected error"),
|
||||
|
||||
Reference in new issue
Block a user