derp,control,logtail: add baseline heartbeat and upload metrics

Count DERP keepalive and ping/pong traffic, terminal read-deadline
expirations, and control map-request watchdog expirations. Count
successful pre-compression log upload bytes alongside existing upload
totals to support batch-size comparisons.

Keep timeout attribution narrow: expirations do not imply a successful
reconnect, and completed writes do not imply peer receipt. Log upload
metrics remain on the existing per-logger ExpVar surface.

Add deterministic coverage without changing heartbeat timing, socket
settings, reconnect behavior, or upload policy.

Updates tailscale/corp#50948

Change-Id: I95f7bab62c000172a4825283bb4c055dd29d8043
Signed-off-by: James Tucker <james@tailscale.com>
This commit is contained in:
James Tucker authored and James Tucker committed 2026-11-10 16:48:38 -08:00
1 parent 4ad0fa1bee
commit 1f6259a88b
6 files changed
+377 -5

No files matched your search

+5
View File
@@ -1187,6 +1187,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap
vlogf("netmap: ending timeout goroutine")
return
case <-watchdogTimedOut:
metricMapRequestWatchdogTimeout.Add(1)
c.logf("map response long-poll timed out!")
cancel()
return
@@ -2004,6 +2005,10 @@ var (
metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite")
metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll")
// Includes both the initial HTTP request and waits for map responses;
// this is not exclusively a streaming keepalive timeout or a reconnect.
metricMapRequestWatchdogTimeout = clientmetric.NewCounter("controlclient_map_request_watchdog_timeout")
metricMapResponseMessages = clientmetric.NewCounter("controlclient_map_response_message") // any message type
metricMapResponsePings = clientmetric.NewCounter("controlclient_map_response_ping")
metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive")
+98
View File
@@ -5,27 +5,125 @@ package controlclient
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"errors"
"io"
"math"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"testing"
"testing/synctest"
"time"
"github.com/klauspost/compress/zstd"
"tailscale.com/feature/buildfeatures"
"tailscale.com/health"
"tailscale.com/hostinfo"
"tailscale.com/ipn/ipnstate"
"tailscale.com/net/netmon"
"tailscale.com/net/tsdial"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
"tailscale.com/types/persist"
"tailscale.com/util/eventbus/eventbustest"
)
func TestMapRequestWatchdogTimeoutMetric(t *testing.T) {
if !buildfeatures.HasClientMetrics {
t.Skip("client metrics disabled")
}
for _, tt := range []struct {
name string
inBody bool
cancel bool
transportErr bool
want int64
}{
{name: "initial_request", want: 1},
{name: "map_response", inBody: true, want: 1},
{name: "canceled_request", cancel: true},
{name: "canceled_response", inBody: true, cancel: true},
{name: "transport_error", transportErr: true},
} {
t.Run(tt.name, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
before := metricMapRequestWatchdogTimeout.Value()
bus := eventbustest.NewBus(t)
dialer := tsdial.NewDialer(netmon.NewStatic())
dialer.SetBus(bus)
defer dialer.Close()
transportErr := errors.New("transport failed")
httpc := &http.Client{Transport: mapWatchdogRoundTripper(func(r *http.Request) (*http.Response, error) {
if tt.transportErr {
return nil, transportErr
}
if tt.inBody {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(mapWatchdogReader{r.Context()}), Header: make(http.Header)}, nil
}
<-r.Context().Done()
return nil, r.Context().Err()
})}
machineKey := key.NewMachine()
c, err := NewDirect(Options{
ServerURL: "https://control.example",
GetMachinePrivateKey: func() (key.MachinePrivate, error) { return machineKey, nil },
Persist: persist.Persist{PrivateNodeKey: key.NewNode()},
Hostinfo: &tailcfg.Hostinfo{BackendLogID: "test-backend-log-id"},
NoiseTestClient: httpc,
HealthTracker: health.NewTracker(bus),
Dialer: dialer,
Bus: bus,
Logf: t.Logf,
})
if err != nil {
t.Fatal(err)
}
defer c.Close()
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
start := time.Now()
go func() { done <- c.sendMapRequest(ctx, true, &countingNetmapUpdater{}) }()
synctest.Wait()
if tt.cancel {
cancel()
}
err = <-done // advances fake time only if the watchdog is needed
if tt.transportErr {
if !errors.Is(err, transportErr) {
t.Fatalf("sendMapRequest error = %v; want transport error", err)
}
} else if !errors.Is(err, context.Canceled) {
t.Fatalf("sendMapRequest error = %v; want context.Canceled", err)
}
synctest.Wait() // let the canceled watchdog goroutine exit
if got := metricMapRequestWatchdogTimeout.Value() - before; got != tt.want {
t.Errorf("watchdog timeout delta = %d; want %d", got, tt.want)
}
wantElapsed := time.Duration(tt.want) * watchdogTimeout
if got := time.Since(start); got != wantElapsed {
t.Errorf("request elapsed = %v; want %v", got, wantElapsed)
}
})
})
}
}
type mapWatchdogRoundTripper func(*http.Request) (*http.Response, error)
func (f mapWatchdogRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
type mapWatchdogReader struct{ ctx context.Context }
func (r mapWatchdogReader) Read([]byte) (int, error) {
<-r.ctx.Done()
return 0, r.ctx.Err()
}
func TestSetDiscoPublicKey(t *testing.T) {
initialKey := key.NewDisco().Public()
+150
View File
@@ -12,20 +12,24 @@ import (
"io"
"net"
"net/netip"
"os"
"reflect"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/iotest"
"testing/synctest"
"time"
"go4.org/mem"
"tailscale.com/feature/buildfeatures"
"tailscale.com/tstest"
"tailscale.com/tstime"
"tailscale.com/types/key"
"tailscale.com/types/logger"
"tailscale.com/util/clientmetric"
)
type dummyNetConn struct {
@@ -103,6 +107,152 @@ func TestClientRecv(t *testing.T) {
}
}
func TestClientRecvMetrics(t *testing.T) {
if !buildfeatures.HasClientMetrics {
t.Skip("client metrics disabled")
}
metrics := []*clientmetric.Metric{metricClientRecvKeepAlive, metricClientRecvPing, metricClientRecvPong}
for _, tt := range []struct {
name string
frame FrameType
payload []byte
want []int64
wantErr bool
}{
{"keepalive", FrameKeepAlive, nil, []int64{1, 0, 0}, false},
{"ping", FramePing, make([]byte, 8), []int64{0, 1, 0}, false},
{"pong", FramePong, make([]byte, 8), []int64{0, 0, 1}, false},
{"short_ping", FramePing, make([]byte, 7), []int64{0, 0, 0}, true},
{"short_pong", FramePong, make([]byte, 7), []int64{0, 0, 0}, true},
{"truncated_ping", FramePing, nil, []int64{0, 0, 0}, true},
} {
t.Run(tt.name, func(t *testing.T) {
before := make([]int64, len(metrics))
for i, m := range metrics {
before[i] = m.Value()
}
var buf bytes.Buffer
bw := bufio.NewWriter(&buf)
if err := WriteFrame(bw, tt.frame, tt.payload); err != nil {
t.Fatal(err)
}
if tt.name == "truncated_ping" {
buf.Bytes()[4] = 8 // header claims a payload that never arrives
}
c := &Client{nc: dummyNetConn{}, br: bufio.NewReader(&buf), logf: t.Logf}
if _, err := c.Recv(); (err != nil) != tt.wantErr {
t.Fatalf("Recv error = %v; wantErr %v", err, tt.wantErr)
}
for i, m := range metrics {
if got := m.Value() - before[i]; got != tt.want[i] {
t.Errorf("receive metric %d delta = %d; want %d", i, got, tt.want[i])
}
}
})
}
}
func TestClientRecvReadTimeoutMetric(t *testing.T) {
if !buildfeatures.HasClientMetrics {
t.Skip("client metrics disabled")
}
t.Run("read_deadline", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
before := metricClientRecvReadTimeout.Value()
nc, peer := net.Pipe()
defer nc.Close()
defer peer.Close()
c := &Client{nc: nc, br: bufio.NewReader(nc)}
start := time.Now()
if _, err := c.Recv(); !errors.Is(err, os.ErrDeadlineExceeded) {
t.Fatalf("Recv error = %v; want read deadline", err)
}
if got := time.Since(start); got != 120*time.Second {
t.Errorf("Recv elapsed = %v; want 120s", got)
}
if got := metricClientRecvReadTimeout.Value() - before; got != 1 {
t.Errorf("read timeout delta = %d; want 1", got)
}
})
})
for _, tt := range []struct {
name string
err error
want int64
}{
{"deadline", os.ErrDeadlineExceeded, 1},
{"wrapped_deadline", fmt.Errorf("read: %w", os.ErrDeadlineExceeded), 1},
{"eof", io.EOF, 0},
{"closed", net.ErrClosed, 0},
{"other_timeout", recvTestTimeoutError{}, 0},
} {
t.Run(tt.name, func(t *testing.T) {
before := metricClientRecvReadTimeout.Value()
c := &Client{nc: dummyNetConn{}, br: bufio.NewReader(iotest.ErrReader(tt.err))}
_, firstErr := c.Recv()
if !errors.Is(firstErr, tt.err) {
t.Fatalf("Recv error = %v; want wrapping %v", firstErr, tt.err)
}
if _, err := c.Recv(); err != firstErr {
t.Fatalf("second Recv error = %v; want sticky error %v", err, firstErr)
}
if got := metricClientRecvReadTimeout.Value() - before; got != tt.want {
t.Errorf("read timeout delta = %d; want %d", got, tt.want)
}
})
}
}
// A generic net.Error timeout must not be attributed to the read deadline.
// In particular, a transport failure is not necessarily a receive-idle expiry.
type recvTestTimeoutError struct{}
func (recvTestTimeoutError) Error() string { return "other timeout" }
func (recvTestTimeoutError) Timeout() bool { return true }
func (recvTestTimeoutError) Temporary() bool { return true }
func TestClientSendPingPongMetrics(t *testing.T) {
if !buildfeatures.HasClientMetrics {
t.Skip("client metrics disabled")
}
for _, frame := range []FrameType{FramePing, FramePong} {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprintf("frame_%x/fail_%v", frame, fail), func(t *testing.T) {
beforePing, beforePong := metricClientSendPing.Value(), metricClientSendPong.Value()
var buf bytes.Buffer
c := &Client{bw: bufio.NewWriter(&buf)}
if fail {
// A buffered writer retains its first error. Neither a failed
// flush nor an already broken writer should count a send.
pr, pw := io.Pipe()
pr.CloseWithError(io.ErrClosedPipe)
defer pw.Close()
c.bw = bufio.NewWriter(pw)
}
for range 2 {
if err := c.sendPingOrPong(frame, [8]byte{}); (err != nil) != fail {
t.Fatalf("send error = %v; want failure %v", err, fail)
}
}
var wantPing, wantPong int64
if !fail {
if frame == FramePing {
wantPing = 2
} else {
wantPong = 2
}
}
if got := metricClientSendPing.Value() - beforePing; got != wantPing {
t.Errorf("sent ping delta = %d; want %d", got, wantPing)
}
if got := metricClientSendPong.Value() - beforePong; got != wantPong {
t.Errorf("sent pong delta = %d; want %d", got, wantPong)
}
})
}
}
}
// TestClientRecvPeerPresent tests that the client can parse peerPresent
// frames from servers of various eras: old servers that send fewer fields
// than the client knows about, and newer servers that send trailing fields
+34 -1
View File
@@ -12,6 +12,7 @@ import (
"io"
"net"
"net/netip"
"os"
"sync"
"time"
@@ -21,6 +22,23 @@ import (
"tailscale.com/tstime"
"tailscale.com/types/key"
"tailscale.com/types/logger"
"tailscale.com/util/clientmetric"
)
var (
// Counts messages returned by Recv, including pongs consumed by derphttp's
// Ping helper. These are process-wide across all DERP clients, including
// mesh clients and probers, not per-connection or purpose-specific.
metricClientRecvKeepAlive = clientmetric.NewCounter("derp_client_recv_keepalive")
metricClientRecvPing = clientmetric.NewCounter("derp_client_recv_ping")
metricClientRecvPong = clientmetric.NewCounter("derp_client_recv_pong")
metricClientSendPing = clientmetric.NewCounter("derp_client_send_ping")
metricClientSendPong = clientmetric.NewCounter("derp_client_send_pong")
// Counts read-deadline expirations once per dead client, not subsequent
// Recv calls returning the sticky error. This includes incomplete frames,
// not just connections on which no bytes were received.
metricClientRecvReadTimeout = clientmetric.NewCounter("derp_client_recv_read_timeout")
)
// Client is a DERP client.
@@ -375,7 +393,16 @@ func (c *Client) sendPingOrPong(typ FrameType, data [8]byte) error {
if _, err := c.bw.Write(data[:]); err != nil {
return err
}
return c.bw.Flush()
if err := c.bw.Flush(); err != nil {
return err
}
// Count completed writes, not attempts or acknowledgements by the peer.
if typ == FramePing {
metricClientSendPing.Add(1)
} else {
metricClientSendPong.Add(1)
}
return nil
}
// NotePreferred sends a packet that tells the server whether this
@@ -567,6 +594,9 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
}
defer func() {
if err != nil {
if errors.Is(err, os.ErrDeadlineExceeded) {
metricClientRecvReadTimeout.Add(1)
}
err = fmt.Errorf("derp.Recv: %w", err)
c.readErr.Store(err)
}
@@ -634,6 +664,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
case FrameKeepAlive:
// A one-way keep-alive message that doesn't require an acknowledgement.
// This predated FramePing/FramePong.
metricClientRecvKeepAlive.Add(1)
return KeepAliveMessage{}, nil
case FramePeerGone:
if n < KeyLen {
@@ -713,6 +744,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
continue
}
copy(pm[:], b[:])
metricClientRecvPing.Add(1)
return pm, nil
case FramePong:
@@ -722,6 +754,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro
continue
}
copy(pm[:], b[:])
metricClientRecvPong.Add(1)
return pm, nil
case FrameHealth:
+15 -4
View File
@@ -331,10 +331,11 @@ type Logger struct {
shutdownDone chan struct{} // closed when shutdown complete
// Metrics (see [Logger.ExpVar] for details).
uploadCalls expvar.Int
failedCalls expvar.Int
uploadedBytes expvar.Int
uploadingTime expvar.Int
uploadCalls expvar.Int
failedCalls expvar.Int
uploadedBytes expvar.Int
uploadedUncompressedBytes expvar.Int
uploadingTime expvar.Int
}
type atomicSocktatsLabel struct{ p atomic.Uint32 }
@@ -686,6 +687,11 @@ func (lg *Logger) upload(ctx context.Context, body []byte, origlen int) (retryAf
}
lg.uploadedBytes.Add(int64(len(body)))
if origlen == -1 {
lg.uploadedUncompressedBytes.Add(int64(len(body)))
} else {
lg.uploadedUncompressedBytes.Add(int64(origlen))
}
lg.uploadingTime.Add(int64(time.Since(startUpload)))
return 0, nil
}
@@ -717,6 +723,10 @@ func (lg *Logger) StartFlush() {
// - counter_uploaded_bytes: Total number of bytes successfully uploaded
// (which is calculated after compression is applied).
//
// - counter_uploaded_uncompressed_bytes: Total pre-compression body bytes
// successfully uploaded. When no upload is in flight, divide by successful
// calls (upload calls minus errors) to obtain the mean uncompressed batch size.
//
// - counter_uploading_nsecs: Total number of nanoseconds spent uploading.
//
// - buffer: An optional [metrics.Set] with metrics for the [Buffer].
@@ -725,6 +735,7 @@ func (lg *Logger) ExpVar() expvar.Var {
m.Set("counter_upload_calls", &lg.uploadCalls)
m.Set("counter_upload_errors", &lg.failedCalls)
m.Set("counter_uploaded_bytes", &lg.uploadedBytes)
m.Set("counter_uploaded_uncompressed_bytes", &lg.uploadedUncompressedBytes)
m.Set("counter_uploading_nsecs", &lg.uploadingTime)
if v, ok := lg.buffer.(interface{ ExpVar() expvar.Var }); ok {
m.Set("buffer", v.ExpVar())
+75
View File
@@ -7,6 +7,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
@@ -587,6 +588,80 @@ type roundTripperFunc func(*http.Request) (*http.Response, error)
func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestUploadMetrics(t *testing.T) {
lg := newLogger(Config{BaseURL: "https://logtail.example", Stderr: io.Discard})
var wantCalls, wantErrors, wantBytes, wantUncompressedBytes int64
for _, tt := range []struct {
name string
body string
origlen int
status int
transportErr bool
}{
{"uncompressed", `[{"text":"hello"}]`, -1, http.StatusOK, false},
{"compressed", "compressed body", 100, http.StatusOK, false},
{"failed_status", "compressed body", 100, http.StatusServiceUnavailable, false},
{"failed_transport", "compressed body", 100, 0, true},
{"retry_success", "compressed body", 100, http.StatusOK, false},
} {
t.Run(tt.name, func(t *testing.T) {
lg.httpc = &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Fatal(err)
}
if string(body) != tt.body {
t.Errorf("request body = %q; want %q", body, tt.body)
}
wantEncoding, wantOriglen := "", ""
if tt.origlen != -1 {
wantEncoding, wantOriglen = "zstd", fmt.Sprint(tt.origlen)
}
if got := r.Header.Get("Content-Encoding"); got != wantEncoding {
t.Errorf("Content-Encoding = %q; want %q", got, wantEncoding)
}
if got := r.Header.Get("Orig-Content-Length"); got != wantOriglen {
t.Errorf("Orig-Content-Length = %q; want %q", got, wantOriglen)
}
if tt.transportErr {
return nil, errors.New("transport failed")
}
return &http.Response{StatusCode: tt.status, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
})}
_, err := lg.upload(t.Context(), []byte(tt.body), tt.origlen)
wantErr := tt.transportErr || tt.status != http.StatusOK
if (err != nil) != wantErr {
t.Fatalf("upload error = %v; wantErr %v", err, wantErr)
}
wantCalls++
if wantErr {
wantErrors++
} else {
wantBytes += int64(len(tt.body))
if tt.origlen == -1 {
wantUncompressedBytes += int64(len(tt.body))
} else {
wantUncompressedBytes += int64(tt.origlen)
}
}
var got map[string]json.RawMessage
if err := json.Unmarshal([]byte(lg.ExpVar().String()), &got); err != nil {
t.Fatal(err)
}
for name, want := range map[string]int64{
"counter_upload_calls": wantCalls,
"counter_upload_errors": wantErrors,
"counter_uploaded_bytes": wantBytes,
"counter_uploaded_uncompressed_bytes": wantUncompressedBytes,
} {
if string(got[name]) != fmt.Sprint(want) {
t.Errorf("%s = %s; want %d", name, got[name], want)
}
}
})
}
}
func TestNewLoggerDisabled(t *testing.T) { synctest.Test(t, synctestNewLoggerDisabled) }
func synctestNewLoggerDisabled(t *testing.T) {