From 1f6259a88bfead9add49c1acabbaa0b4cf7efbe0 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 10 Nov 2026 20:18:45 +0000 Subject: [PATCH] derp,control,logtail: add baseline heartbeat and upload metrics Count DERP keepalive and ping/pong traffic, terminal read-deadline expirations, and control map-request watchdog expirations. Count successful pre-compression log upload bytes alongside existing upload totals to support batch-size comparisons. Keep timeout attribution narrow: expirations do not imply a successful reconnect, and completed writes do not imply peer receipt. Log upload metrics remain on the existing per-logger ExpVar surface. Add deterministic coverage without changing heartbeat timing, socket settings, reconnect behavior, or upload policy. Updates tailscale/corp#50948 Change-Id: I95f7bab62c000172a4825283bb4c055dd29d8043 Signed-off-by: James Tucker --- control/controlclient/direct.go | 5 + control/controlclient/direct_test.go | 98 +++++++++++++++++ derp/client_test.go | 150 +++++++++++++++++++++++++++ derp/derp_client.go | 35 ++++++- logtail/logtail.go | 19 +++- logtail/logtail_test.go | 75 ++++++++++++++ 6 files changed, 377 insertions(+), 5 deletions(-) diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index 2405881fb..a6ef105a5 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -1187,6 +1187,7 @@ func (c *Direct) sendMapRequest(ctx context.Context, isStreaming bool, nu Netmap vlogf("netmap: ending timeout goroutine") return case <-watchdogTimedOut: + metricMapRequestWatchdogTimeout.Add(1) c.logf("map response long-poll timed out!") cancel() return @@ -2004,6 +2005,10 @@ var ( metricMapRequestsLite = clientmetric.NewCounter("controlclient_map_requests_lite") metricMapRequestsPoll = clientmetric.NewCounter("controlclient_map_requests_poll") + // Includes both the initial HTTP request and waits for map responses; + // this is not exclusively a streaming keepalive timeout or a reconnect. + metricMapRequestWatchdogTimeout = clientmetric.NewCounter("controlclient_map_request_watchdog_timeout") + metricMapResponseMessages = clientmetric.NewCounter("controlclient_map_response_message") // any message type metricMapResponsePings = clientmetric.NewCounter("controlclient_map_response_ping") metricMapResponseKeepAlives = clientmetric.NewCounter("controlclient_map_response_keepalive") diff --git a/control/controlclient/direct_test.go b/control/controlclient/direct_test.go index 46fe2afcf..cf0bb3e68 100644 --- a/control/controlclient/direct_test.go +++ b/control/controlclient/direct_test.go @@ -5,27 +5,125 @@ package controlclient import ( "bytes" + "context" "encoding/binary" "encoding/json" "errors" + "io" "math" "net/http" "net/http/httptest" "net/netip" "strings" "testing" + "testing/synctest" "time" "github.com/klauspost/compress/zstd" + "tailscale.com/feature/buildfeatures" + "tailscale.com/health" "tailscale.com/hostinfo" "tailscale.com/ipn/ipnstate" "tailscale.com/net/netmon" "tailscale.com/net/tsdial" "tailscale.com/tailcfg" "tailscale.com/types/key" + "tailscale.com/types/persist" "tailscale.com/util/eventbus/eventbustest" ) +func TestMapRequestWatchdogTimeoutMetric(t *testing.T) { + if !buildfeatures.HasClientMetrics { + t.Skip("client metrics disabled") + } + for _, tt := range []struct { + name string + inBody bool + cancel bool + transportErr bool + want int64 + }{ + {name: "initial_request", want: 1}, + {name: "map_response", inBody: true, want: 1}, + {name: "canceled_request", cancel: true}, + {name: "canceled_response", inBody: true, cancel: true}, + {name: "transport_error", transportErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + before := metricMapRequestWatchdogTimeout.Value() + bus := eventbustest.NewBus(t) + dialer := tsdial.NewDialer(netmon.NewStatic()) + dialer.SetBus(bus) + defer dialer.Close() + transportErr := errors.New("transport failed") + httpc := &http.Client{Transport: mapWatchdogRoundTripper(func(r *http.Request) (*http.Response, error) { + if tt.transportErr { + return nil, transportErr + } + if tt.inBody { + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(mapWatchdogReader{r.Context()}), Header: make(http.Header)}, nil + } + <-r.Context().Done() + return nil, r.Context().Err() + })} + machineKey := key.NewMachine() + c, err := NewDirect(Options{ + ServerURL: "https://control.example", + GetMachinePrivateKey: func() (key.MachinePrivate, error) { return machineKey, nil }, + Persist: persist.Persist{PrivateNodeKey: key.NewNode()}, + Hostinfo: &tailcfg.Hostinfo{BackendLogID: "test-backend-log-id"}, + NoiseTestClient: httpc, + HealthTracker: health.NewTracker(bus), + Dialer: dialer, + Bus: bus, + Logf: t.Logf, + }) + if err != nil { + t.Fatal(err) + } + defer c.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + start := time.Now() + go func() { done <- c.sendMapRequest(ctx, true, &countingNetmapUpdater{}) }() + synctest.Wait() + if tt.cancel { + cancel() + } + err = <-done // advances fake time only if the watchdog is needed + if tt.transportErr { + if !errors.Is(err, transportErr) { + t.Fatalf("sendMapRequest error = %v; want transport error", err) + } + } else if !errors.Is(err, context.Canceled) { + t.Fatalf("sendMapRequest error = %v; want context.Canceled", err) + } + synctest.Wait() // let the canceled watchdog goroutine exit + if got := metricMapRequestWatchdogTimeout.Value() - before; got != tt.want { + t.Errorf("watchdog timeout delta = %d; want %d", got, tt.want) + } + wantElapsed := time.Duration(tt.want) * watchdogTimeout + if got := time.Since(start); got != wantElapsed { + t.Errorf("request elapsed = %v; want %v", got, wantElapsed) + } + }) + }) + } +} + +type mapWatchdogRoundTripper func(*http.Request) (*http.Response, error) + +func (f mapWatchdogRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type mapWatchdogReader struct{ ctx context.Context } + +func (r mapWatchdogReader) Read([]byte) (int, error) { + <-r.ctx.Done() + return 0, r.ctx.Err() +} + func TestSetDiscoPublicKey(t *testing.T) { initialKey := key.NewDisco().Public() diff --git a/derp/client_test.go b/derp/client_test.go index fc83a95cc..df796ce26 100644 --- a/derp/client_test.go +++ b/derp/client_test.go @@ -12,20 +12,24 @@ import ( "io" "net" "net/netip" + "os" "reflect" "runtime" "strings" "sync" "sync/atomic" "testing" + "testing/iotest" "testing/synctest" "time" "go4.org/mem" + "tailscale.com/feature/buildfeatures" "tailscale.com/tstest" "tailscale.com/tstime" "tailscale.com/types/key" "tailscale.com/types/logger" + "tailscale.com/util/clientmetric" ) type dummyNetConn struct { @@ -103,6 +107,152 @@ func TestClientRecv(t *testing.T) { } } +func TestClientRecvMetrics(t *testing.T) { + if !buildfeatures.HasClientMetrics { + t.Skip("client metrics disabled") + } + metrics := []*clientmetric.Metric{metricClientRecvKeepAlive, metricClientRecvPing, metricClientRecvPong} + for _, tt := range []struct { + name string + frame FrameType + payload []byte + want []int64 + wantErr bool + }{ + {"keepalive", FrameKeepAlive, nil, []int64{1, 0, 0}, false}, + {"ping", FramePing, make([]byte, 8), []int64{0, 1, 0}, false}, + {"pong", FramePong, make([]byte, 8), []int64{0, 0, 1}, false}, + {"short_ping", FramePing, make([]byte, 7), []int64{0, 0, 0}, true}, + {"short_pong", FramePong, make([]byte, 7), []int64{0, 0, 0}, true}, + {"truncated_ping", FramePing, nil, []int64{0, 0, 0}, true}, + } { + t.Run(tt.name, func(t *testing.T) { + before := make([]int64, len(metrics)) + for i, m := range metrics { + before[i] = m.Value() + } + var buf bytes.Buffer + bw := bufio.NewWriter(&buf) + if err := WriteFrame(bw, tt.frame, tt.payload); err != nil { + t.Fatal(err) + } + if tt.name == "truncated_ping" { + buf.Bytes()[4] = 8 // header claims a payload that never arrives + } + c := &Client{nc: dummyNetConn{}, br: bufio.NewReader(&buf), logf: t.Logf} + if _, err := c.Recv(); (err != nil) != tt.wantErr { + t.Fatalf("Recv error = %v; wantErr %v", err, tt.wantErr) + } + for i, m := range metrics { + if got := m.Value() - before[i]; got != tt.want[i] { + t.Errorf("receive metric %d delta = %d; want %d", i, got, tt.want[i]) + } + } + }) + } +} + +func TestClientRecvReadTimeoutMetric(t *testing.T) { + if !buildfeatures.HasClientMetrics { + t.Skip("client metrics disabled") + } + t.Run("read_deadline", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + before := metricClientRecvReadTimeout.Value() + nc, peer := net.Pipe() + defer nc.Close() + defer peer.Close() + c := &Client{nc: nc, br: bufio.NewReader(nc)} + start := time.Now() + if _, err := c.Recv(); !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("Recv error = %v; want read deadline", err) + } + if got := time.Since(start); got != 120*time.Second { + t.Errorf("Recv elapsed = %v; want 120s", got) + } + if got := metricClientRecvReadTimeout.Value() - before; got != 1 { + t.Errorf("read timeout delta = %d; want 1", got) + } + }) + }) + for _, tt := range []struct { + name string + err error + want int64 + }{ + {"deadline", os.ErrDeadlineExceeded, 1}, + {"wrapped_deadline", fmt.Errorf("read: %w", os.ErrDeadlineExceeded), 1}, + {"eof", io.EOF, 0}, + {"closed", net.ErrClosed, 0}, + {"other_timeout", recvTestTimeoutError{}, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + before := metricClientRecvReadTimeout.Value() + c := &Client{nc: dummyNetConn{}, br: bufio.NewReader(iotest.ErrReader(tt.err))} + _, firstErr := c.Recv() + if !errors.Is(firstErr, tt.err) { + t.Fatalf("Recv error = %v; want wrapping %v", firstErr, tt.err) + } + if _, err := c.Recv(); err != firstErr { + t.Fatalf("second Recv error = %v; want sticky error %v", err, firstErr) + } + if got := metricClientRecvReadTimeout.Value() - before; got != tt.want { + t.Errorf("read timeout delta = %d; want %d", got, tt.want) + } + }) + } +} + +// A generic net.Error timeout must not be attributed to the read deadline. +// In particular, a transport failure is not necessarily a receive-idle expiry. +type recvTestTimeoutError struct{} + +func (recvTestTimeoutError) Error() string { return "other timeout" } +func (recvTestTimeoutError) Timeout() bool { return true } +func (recvTestTimeoutError) Temporary() bool { return true } + +func TestClientSendPingPongMetrics(t *testing.T) { + if !buildfeatures.HasClientMetrics { + t.Skip("client metrics disabled") + } + for _, frame := range []FrameType{FramePing, FramePong} { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("frame_%x/fail_%v", frame, fail), func(t *testing.T) { + beforePing, beforePong := metricClientSendPing.Value(), metricClientSendPong.Value() + var buf bytes.Buffer + c := &Client{bw: bufio.NewWriter(&buf)} + if fail { + // A buffered writer retains its first error. Neither a failed + // flush nor an already broken writer should count a send. + pr, pw := io.Pipe() + pr.CloseWithError(io.ErrClosedPipe) + defer pw.Close() + c.bw = bufio.NewWriter(pw) + } + for range 2 { + if err := c.sendPingOrPong(frame, [8]byte{}); (err != nil) != fail { + t.Fatalf("send error = %v; want failure %v", err, fail) + } + } + var wantPing, wantPong int64 + if !fail { + if frame == FramePing { + wantPing = 2 + } else { + wantPong = 2 + } + } + if got := metricClientSendPing.Value() - beforePing; got != wantPing { + t.Errorf("sent ping delta = %d; want %d", got, wantPing) + } + if got := metricClientSendPong.Value() - beforePong; got != wantPong { + t.Errorf("sent pong delta = %d; want %d", got, wantPong) + } + }) + } + } +} + // TestClientRecvPeerPresent tests that the client can parse peerPresent // frames from servers of various eras: old servers that send fewer fields // than the client knows about, and newer servers that send trailing fields diff --git a/derp/derp_client.go b/derp/derp_client.go index 6f94e3930..bfd89b806 100644 --- a/derp/derp_client.go +++ b/derp/derp_client.go @@ -12,6 +12,7 @@ import ( "io" "net" "net/netip" + "os" "sync" "time" @@ -21,6 +22,23 @@ import ( "tailscale.com/tstime" "tailscale.com/types/key" "tailscale.com/types/logger" + "tailscale.com/util/clientmetric" +) + +var ( + // Counts messages returned by Recv, including pongs consumed by derphttp's + // Ping helper. These are process-wide across all DERP clients, including + // mesh clients and probers, not per-connection or purpose-specific. + metricClientRecvKeepAlive = clientmetric.NewCounter("derp_client_recv_keepalive") + metricClientRecvPing = clientmetric.NewCounter("derp_client_recv_ping") + metricClientRecvPong = clientmetric.NewCounter("derp_client_recv_pong") + metricClientSendPing = clientmetric.NewCounter("derp_client_send_ping") + metricClientSendPong = clientmetric.NewCounter("derp_client_send_pong") + + // Counts read-deadline expirations once per dead client, not subsequent + // Recv calls returning the sticky error. This includes incomplete frames, + // not just connections on which no bytes were received. + metricClientRecvReadTimeout = clientmetric.NewCounter("derp_client_recv_read_timeout") ) // Client is a DERP client. @@ -375,7 +393,16 @@ func (c *Client) sendPingOrPong(typ FrameType, data [8]byte) error { if _, err := c.bw.Write(data[:]); err != nil { return err } - return c.bw.Flush() + if err := c.bw.Flush(); err != nil { + return err + } + // Count completed writes, not attempts or acknowledgements by the peer. + if typ == FramePing { + metricClientSendPing.Add(1) + } else { + metricClientSendPong.Add(1) + } + return nil } // NotePreferred sends a packet that tells the server whether this @@ -567,6 +594,9 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro } defer func() { if err != nil { + if errors.Is(err, os.ErrDeadlineExceeded) { + metricClientRecvReadTimeout.Add(1) + } err = fmt.Errorf("derp.Recv: %w", err) c.readErr.Store(err) } @@ -634,6 +664,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro case FrameKeepAlive: // A one-way keep-alive message that doesn't require an acknowledgement. // This predated FramePing/FramePong. + metricClientRecvKeepAlive.Add(1) return KeepAliveMessage{}, nil case FramePeerGone: if n < KeyLen { @@ -713,6 +744,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro continue } copy(pm[:], b[:]) + metricClientRecvPing.Add(1) return pm, nil case FramePong: @@ -722,6 +754,7 @@ func (c *Client) recvTimeout(timeout time.Duration) (m ReceivedMessage, err erro continue } copy(pm[:], b[:]) + metricClientRecvPong.Add(1) return pm, nil case FrameHealth: diff --git a/logtail/logtail.go b/logtail/logtail.go index 9e7dc228d..860488a86 100644 --- a/logtail/logtail.go +++ b/logtail/logtail.go @@ -331,10 +331,11 @@ type Logger struct { shutdownDone chan struct{} // closed when shutdown complete // Metrics (see [Logger.ExpVar] for details). - uploadCalls expvar.Int - failedCalls expvar.Int - uploadedBytes expvar.Int - uploadingTime expvar.Int + uploadCalls expvar.Int + failedCalls expvar.Int + uploadedBytes expvar.Int + uploadedUncompressedBytes expvar.Int + uploadingTime expvar.Int } type atomicSocktatsLabel struct{ p atomic.Uint32 } @@ -686,6 +687,11 @@ func (lg *Logger) upload(ctx context.Context, body []byte, origlen int) (retryAf } lg.uploadedBytes.Add(int64(len(body))) + if origlen == -1 { + lg.uploadedUncompressedBytes.Add(int64(len(body))) + } else { + lg.uploadedUncompressedBytes.Add(int64(origlen)) + } lg.uploadingTime.Add(int64(time.Since(startUpload))) return 0, nil } @@ -717,6 +723,10 @@ func (lg *Logger) StartFlush() { // - counter_uploaded_bytes: Total number of bytes successfully uploaded // (which is calculated after compression is applied). // +// - counter_uploaded_uncompressed_bytes: Total pre-compression body bytes +// successfully uploaded. When no upload is in flight, divide by successful +// calls (upload calls minus errors) to obtain the mean uncompressed batch size. +// // - counter_uploading_nsecs: Total number of nanoseconds spent uploading. // // - buffer: An optional [metrics.Set] with metrics for the [Buffer]. @@ -725,6 +735,7 @@ func (lg *Logger) ExpVar() expvar.Var { m.Set("counter_upload_calls", &lg.uploadCalls) m.Set("counter_upload_errors", &lg.failedCalls) m.Set("counter_uploaded_bytes", &lg.uploadedBytes) + m.Set("counter_uploaded_uncompressed_bytes", &lg.uploadedUncompressedBytes) m.Set("counter_uploading_nsecs", &lg.uploadingTime) if v, ok := lg.buffer.(interface{ ExpVar() expvar.Var }); ok { m.Set("buffer", v.ExpVar()) diff --git a/logtail/logtail_test.go b/logtail/logtail_test.go index 44182880b..c72733473 100644 --- a/logtail/logtail_test.go +++ b/logtail/logtail_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net" @@ -587,6 +588,80 @@ type roundTripperFunc func(*http.Request) (*http.Response, error) func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } +func TestUploadMetrics(t *testing.T) { + lg := newLogger(Config{BaseURL: "https://logtail.example", Stderr: io.Discard}) + var wantCalls, wantErrors, wantBytes, wantUncompressedBytes int64 + for _, tt := range []struct { + name string + body string + origlen int + status int + transportErr bool + }{ + {"uncompressed", `[{"text":"hello"}]`, -1, http.StatusOK, false}, + {"compressed", "compressed body", 100, http.StatusOK, false}, + {"failed_status", "compressed body", 100, http.StatusServiceUnavailable, false}, + {"failed_transport", "compressed body", 100, 0, true}, + {"retry_success", "compressed body", 100, http.StatusOK, false}, + } { + t.Run(tt.name, func(t *testing.T) { + lg.httpc = &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != tt.body { + t.Errorf("request body = %q; want %q", body, tt.body) + } + wantEncoding, wantOriglen := "", "" + if tt.origlen != -1 { + wantEncoding, wantOriglen = "zstd", fmt.Sprint(tt.origlen) + } + if got := r.Header.Get("Content-Encoding"); got != wantEncoding { + t.Errorf("Content-Encoding = %q; want %q", got, wantEncoding) + } + if got := r.Header.Get("Orig-Content-Length"); got != wantOriglen { + t.Errorf("Orig-Content-Length = %q; want %q", got, wantOriglen) + } + if tt.transportErr { + return nil, errors.New("transport failed") + } + return &http.Response{StatusCode: tt.status, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil + })} + _, err := lg.upload(t.Context(), []byte(tt.body), tt.origlen) + wantErr := tt.transportErr || tt.status != http.StatusOK + if (err != nil) != wantErr { + t.Fatalf("upload error = %v; wantErr %v", err, wantErr) + } + wantCalls++ + if wantErr { + wantErrors++ + } else { + wantBytes += int64(len(tt.body)) + if tt.origlen == -1 { + wantUncompressedBytes += int64(len(tt.body)) + } else { + wantUncompressedBytes += int64(tt.origlen) + } + } + var got map[string]json.RawMessage + if err := json.Unmarshal([]byte(lg.ExpVar().String()), &got); err != nil { + t.Fatal(err) + } + for name, want := range map[string]int64{ + "counter_upload_calls": wantCalls, + "counter_upload_errors": wantErrors, + "counter_uploaded_bytes": wantBytes, + "counter_uploaded_uncompressed_bytes": wantUncompressedBytes, + } { + if string(got[name]) != fmt.Sprint(want) { + t.Errorf("%s = %s; want %d", name, got[name], want) + } + } + }) + } +} + func TestNewLoggerDisabled(t *testing.T) { synctest.Test(t, synctestNewLoggerDisabled) } func synctestNewLoggerDisabled(t *testing.T) {