tsweb/varz: read memstats via runtime/metrics, export fork metrics

The varz handler got its memstats_* metrics from the expvar package's
"memstats" func, which calls runtime.ReadMemStats and so stopped the
world on every Prometheus scrape. Keep the names but compute them from
runtime/metrics, and never call that func, even from
WritePrometheusExpvar.

While there, export the /tailscale/ metrics from our Go fork (stack
size histogram, stack copy counters, timer zombie counts and lifetime
histogram), which nothing could see before, plus a few upstream ones
with no MemStats equivalent: scheduling latency and GC pause
histograms, live heap, GC and total CPU seconds, thread count, and
mutex wait time. The last replaces derper's hand-rolled version.

Everything read is cheap and a scrape allocates nothing after the
first. Names use a go_runtime_ namespace rather than go_ so they can't
collide with the Prometheus Go client's collector in promvarz binaries.
The runtime's 162-bucket time histograms are reduced to one bucket per
factor of four from 256ns to 1s.

Updates #21300
Updates tailscale/go#189
Updates golang/go#75935

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
Change-Id: I7e3c9a41f2b85d6e0c4a9b1d3f8e7c2a5b6d4e19

cmd/derper/derper.go

@@ -32,7 +32,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
-	runtimemetrics "runtime/metrics"
 	"strconv"
 	"strings"
 	"sync"
@@ -550,19 +549,6 @@ func (ln *rateLimitedListener) Accept() (net.Conn, error) {
 	return cn, nil
 }
 
-func init() {
-	expvar.Publish("go_sync_mutex_wait_seconds", expvar.Func(func() any {
-		const name = "/sync/mutex/wait/total:seconds" // Go 1.20+
-		var s [1]runtimemetrics.Sample
-		s[0].Name = name
-		runtimemetrics.Read(s[:])
-		if v := s[0].Value; v.Kind() == runtimemetrics.KindFloat64 {
-			return v.Float64()
-		}
-		return 0
-	}))
-}
-
 type templateData struct {
 	ShowAbuseInfo bool
 	Disabled      bool

cmd/k8s-operator/depaware.txt

@@ -960,7 +960,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from tailscale.com/tsweb/varz+
+        golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from tailscale.com/ipn/store/mem+
         golang.org/x/exp/slices                                      from tailscale.com/cmd/k8s-operator+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+

cmd/stund/depaware.txt

@@ -96,7 +96,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from tailscale.com/tsweb/varz+
+        golang.org/x/exp/constraints                                 from tailscale.com/util/set
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/prometheus/procfs+
    W    golang.org/x/sys/windows                                     from github.com/prometheus/client_golang/prometheus

cmd/tailscale/depaware.txt

@@ -617,7 +617,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         regexp/syntax                                                from regexp
         runtime                                                      from archive/tar+
         runtime/debug                                                from tailscale.com+
-        runtime/metrics                                              from tailscale.com/tsweb
+        runtime/metrics                                              from tailscale.com/tsweb+
         runtime/pprof                                                from net/http/pprof
         runtime/trace                                                from net/http/pprof
         slices                                                       from tailscale.com/client/web+

cmd/tsidp/depaware.txt

@@ -348,7 +348,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar
         golang.org/x/crypto/pbkdf2                                   from gopkg.in/square/go-jose.v2
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from tailscale.com/tsweb/varz+
+        golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from tailscale.com/ipn/store/mem+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from tailscale.com/appc+
@@ -592,7 +592,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar
         regexp/syntax                                                from regexp
         runtime                                                      from crypto/internal/fips140+
         runtime/debug                                                from github.com/klauspost/compress/zstd+
-        runtime/metrics                                              from tailscale.com/tsweb
+        runtime/metrics                                              from tailscale.com/tsweb+
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from crypto/tls+

tsnet/depaware.txt

@@ -341,7 +341,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware)
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-        golang.org/x/exp/constraints                                 from tailscale.com/tsweb/varz+
+        golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from tailscale.com/ipn/store/mem+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from tailscale.com/appc+
@@ -582,7 +582,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware)
         regexp/syntax                                                from regexp
         runtime                                                      from crypto/internal/fips140+
         runtime/debug                                                from github.com/klauspost/compress/zstd+
-        runtime/metrics                                              from tailscale.com/tsweb
+        runtime/metrics                                              from tailscale.com/tsweb+
         runtime/pprof                                                from net/http/pprof+
         runtime/trace                                                from net/http/pprof
         slices                                                       from crypto/tls+

tsweb/varz/runtimemetrics.go

@@ -0,0 +1,508 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package varz
+
+import (
+	"io"
+	"math"
+	"runtime/metrics"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// runtimeMetricSpec describes one Go runtime/metrics metric that
+// [Handler] exports in Prometheus format.
+type runtimeMetricSpec struct {
+	// name is the runtime/metrics name, such as "/gc/heap/allocs:bytes".
+	name string
+
+	// typ is the Prometheus type: "gauge", "counter", or "histogram".
+	// Counters get a "_total" suffix, as in the Prometheus naming
+	// conventions.
+	typ string
+
+	// buckets says how a histogram's runtime buckets map to Prometheus
+	// buckets. It is required for histograms and ignored otherwise.
+	buckets histBuckets
+
+	// hidden means the metric is read (because it backs one of the
+	// legacy memstats_* metrics) but is not exported under its own
+	// go_runtime_ name.
+	hidden bool
+}
+
+// histBuckets says how the runtime's lower-bound-inclusive buckets
+// [lo, hi) of a histogram become Prometheus's upper-bound-inclusive
+// le buckets.
+type histBuckets uint8
+
+const (
+	// histTime is for the runtime's time histograms, which have
+	// hundreds of buckets. They are reduced as described at
+	// [runtimeHistMinBucket], and a runtime bucket [lo, hi) becomes
+	// le=hi, which is only wrong for a value exactly equal to hi.
+	histTime histBuckets = iota + 1
+
+	// histPowersOfTwo is for histograms whose observed values are all
+	// exactly equal to bucket lower bounds, such as goroutine stack
+	// sizes. A runtime bucket [lo, hi) becomes le=lo, so the labels
+	// name the values they count, and the runtime's final catch-all
+	// bucket is folded into +Inf.
+	histPowersOfTwo
+
+	// histInts is for histograms of small integer values, such as
+	// counts of GC cycles. A runtime bucket [lo, hi) becomes le=hi-1,
+	// which is exact for integers.
+	histInts
+)
+
+// runtimeMetricSpecs is the set of Go runtime/metrics metrics that
+// [Handler] reads. Everything here is cheap for the runtime to read: no
+// metric in this list stops the world or walks the heap or the
+// goroutine list.
+//
+// The list is deliberately short. The hidden entries only back the
+// legacy memstats_* metrics. The rest are either specific to the
+// Tailscale fork of Go or have no runtime.MemStats equivalent and earn
+// their place on a dashboard.
+//
+// Names that the running toolchain does not know (such as the
+// /tailscale/ metrics when not built with the fork) are dropped on
+// first use, so it is fine for this list to be a superset of what
+// exists.
+//
+// Go's Cumulative metrics are declared as counters here and the rest
+// as gauges; TestRuntimeMetricSpecs checks that against the
+// runtime's own descriptions.
+var runtimeMetricSpecs = []runtimeMetricSpec{
+	// These back the memstats_* metrics; see memstatsIndexes.
+	{name: "/memory/classes/heap/objects:bytes", typ: "gauge", hidden: true},
+	{name: "/gc/heap/allocs:bytes", typ: "counter", hidden: true},
+	{name: "/memory/classes/total:bytes", typ: "gauge", hidden: true},
+	{name: "/gc/heap/allocs:objects", typ: "counter", hidden: true},
+	{name: "/gc/heap/frees:objects", typ: "counter", hidden: true},
+	{name: "/gc/heap/tiny/allocs:objects", typ: "counter", hidden: true},
+	{name: "/gc/cycles/total:gc-cycles", typ: "counter", hidden: true},
+	{name: "/cpu/classes/gc/mark/idle:cpu-seconds", typ: "counter", hidden: true},
+
+	// GC CPU time as counters, so that rate() over a window works.
+	// memstats_gc_cpu_fraction, derived from these two (less the
+	// hidden idle mark time above), is a lifetime average and cannot
+	// show a recent change.
+	{name: "/cpu/classes/gc/total:cpu-seconds", typ: "counter"},
+	{name: "/cpu/classes/total:cpu-seconds", typ: "counter"},
+
+	// Live heap after the last GC. Unlike memstats_heap_alloc it
+	// excludes garbage not yet swept, so it is the number to compare
+	// against GOMEMLIMIT or a container memory limit.
+	{name: "/gc/heap/live:bytes", typ: "gauge"},
+
+	// How long runnable goroutines wait to be scheduled, and how long
+	// GC stop-the-world pauses take. Neither is visible any other way.
+	{name: "/sched/latencies:seconds", typ: "histogram", buckets: histTime},
+	{name: "/sched/pauses/total/gc:seconds", typ: "histogram", buckets: histTime},
+
+	// OS threads owned by the runtime, for spotting thread leaks from
+	// blocking syscalls or cgo.
+	{name: "/sched/threads/total:threads", typ: "gauge"},
+
+	// Time spent blocked on mutexes. cmd/derper used to export this
+	// itself as go_sync_mutex_wait_seconds.
+	{name: "/sync/mutex/wait/total:seconds", typ: "counter"},
+
+	// The /tailscale/ metrics exist only in the Tailscale fork of Go
+	// (github.com/tailscale/go), used when building with the
+	// tailscale_go build tag.
+	{name: "/tailscale/sched/goroutines-by-stack-size:bytes", typ: "histogram", buckets: histPowersOfTwo},
+	{name: "/tailscale/sched/stacks/copied:bytes", typ: "counter"},
+	{name: "/tailscale/sched/stacks/growths:events", typ: "counter"},
+	{name: "/tailscale/sched/stacks/shrinks:events", typ: "counter"},
+	{name: "/tailscale/sched/timers/tracked:timers", typ: "gauge"},
+	{name: "/tailscale/sched/timers/zombies/chan:timers", typ: "gauge"},
+	{name: "/tailscale/sched/timers/zombies/func:timers", typ: "gauge"},
+	{name: "/tailscale/sched/timers/zombies/func/created:timers", typ: "counter"},
+	{name: "/tailscale/sched/timers/zombies/func/removed:timers", typ: "counter"},
+	{name: "/tailscale/sched/timers/zombies/func/lifetime:gc-cycles", typ: "histogram", buckets: histInts},
+	{name: "/tailscale/sched/timers/zombies:timers", typ: "gauge"},
+}
+
+// runtimeMetricPrefix is the namespace prefix of exported runtime
+// metrics. It is deliberately not "go_", which is the namespace used
+// by the Prometheus Go client's collector; binaries that also use that
+// client (see tailscale.com/tsweb/promvarz) would otherwise export
+// some series twice, and Prometheus rejects a scrape containing
+// duplicate series.
+const runtimeMetricPrefix = "go_runtime_"
+
+// runtimeMetricPromName converts a runtime/metrics name such as
+// "/gc/heap/allocs:bytes" to a Prometheus metric name such as
+// "go_runtime_gc_heap_allocs_bytes_total". The conversion follows the
+// same rules as the Prometheus Go client's collector, other than the
+// namespace: the path becomes the name with slashes and dashes
+// replaced by underscores, the unit is appended, and counters get a
+// "_total" suffix.
+func runtimeMetricPromName(name, typ string) string {
+	path, unit, _ := strings.Cut(strings.TrimPrefix(name, "/"), ":")
+	r := strings.NewReplacer("/", "_", "-", "_")
+	unit = strings.ReplaceAll(unit, "/", "_per_")
+	unit = strings.ReplaceAll(unit, "*", "_")
+	s := runtimeMetricPrefix + r.Replace(path) + "_" + r.Replace(unit)
+	if typ == "counter" {
+		s += "_total"
+	}
+	return s
+}
+
+// Time histograms from the runtime have hundreds of buckets, four per
+// power of two from 256ns (finer below that) to about a day. That is
+// far too many series for Prometheus, so they are reduced to one
+// bucket per factor of runtimeHistBucketFactor starting at
+// runtimeHistMinBucket, and everything above runtimeHistMaxBucket is
+// folded into the +Inf bucket. Because the runtime's bucket boundaries
+// are at powers of two times small integers, a factor of 4 starting at
+// 256ns lands exactly on runtime boundaries, so the reduction only
+// merges runtime buckets and never splits one.
+const (
+	runtimeHistMinBucket    = 256e-9 // seconds
+	runtimeHistMaxBucket    = 1.0    // seconds
+	runtimeHistBucketFactor = 4
+)
+
+// runtimeHistLayout describes how a runtime histogram's buckets are
+// merged into the exported Prometheus buckets. It is computed once per
+// metric from the runtime's bucket boundaries, which never change.
+type runtimeHistLayout struct {
+	// lines[i] is the exported line prefix for output bucket i, up to
+	// and including the space before the value:
+	// `name_bucket{le="0.001"} `. The last entry is the +Inf bucket.
+	lines [][]byte
+
+	// ends[i] is the number of leading runtime bucket counts summed
+	// into output bucket i, which is cumulative like Prometheus
+	// buckets. The last entry, for +Inf, is the total number of
+	// runtime buckets.
+	ends []int
+}
+
+// newRuntimeHistLayout computes the export layout for a runtime
+// histogram with the given bucket boundaries (as in
+// [metrics.Float64Histogram.Buckets], so the runtime bucket i counts
+// values in [buckets[i], buckets[i+1])).
+func newRuntimeHistLayout(promName string, buckets []float64, kind histBuckets) *runtimeHistLayout {
+	numCounts := len(buckets) - 1
+	lay := new(runtimeHistLayout)
+	addBucket := func(le float64, end int) {
+		line := append([]byte(promName), `_bucket{le="`...)
+		if math.IsInf(le, 1) {
+			line = append(line, "+Inf"...)
+		} else {
+			line = strconv.AppendFloat(line, le, 'g', -1, 64)
+		}
+		line = append(line, `"} `...)
+		lay.lines = append(lay.lines, line)
+		lay.ends = append(lay.ends, end)
+	}
+	switch kind {
+	case histPowersOfTwo:
+		// All but the final catch-all runtime bucket, each labeled
+		// with its lower bound.
+		for i := 0; i < numCounts-1; i++ {
+			addBucket(buckets[i], i+1)
+		}
+	case histInts:
+		// All but the final catch-all runtime bucket, each labeled
+		// with the largest integer it contains.
+		for i := 1; i < numCounts; i++ {
+			addBucket(buckets[i]-1, i)
+		}
+	case histTime:
+		last := 0.0
+		for i := 1; i < numCounts; i++ {
+			hi := buckets[i]
+			if math.IsInf(hi, 0) || hi < runtimeHistMinBucket {
+				continue
+			}
+			if last > runtimeHistMaxBucket {
+				break
+			}
+			if last == 0 || hi >= last*runtimeHistBucketFactor {
+				addBucket(hi, i)
+				last = hi
+			}
+		}
+	}
+	addBucket(math.Inf(1), numCounts)
+	return lay
+}
+
+// runtimeMetricState is a metric being exported, alongside its
+// [metrics.Sample] in [runtimeMetricsExporter.samples].
+type runtimeMetricState struct {
+	spec runtimeMetricSpec
+
+	// typeLine is the "# TYPE name type\n" line.
+	typeLine []byte
+
+	// valuePrefix is "name " for scalar metrics, or nil for histograms.
+	valuePrefix []byte
+
+	// countPrefix is "name_count " for histograms, or nil otherwise.
+	countPrefix []byte
+
+	// hist is the bucket layout for histograms, or nil otherwise.
+	hist *runtimeHistLayout
+}
+
+// runtimeMetricsExporter reads a fixed set of Go runtime/metrics and
+// writes them in Prometheus format. There is one, [runtimeMetrics],
+// and it keeps its sample slice across reads so that the runtime can
+// reuse histogram buffers and a steady-state scrape does not allocate.
+type runtimeMetricsExporter struct {
+	mu      sync.Mutex
+	inited  bool
+	samples []metrics.Sample      // reused across reads; parallel to states
+	states  []*runtimeMetricState // parallel to samples
+
+	// Indexes into samples of the metrics backing the legacy
+	// memstats_* names, or -1 if the runtime lacks them.
+	memstatsIdx memstatsIndexes
+}
+
+// memstatsIndexes holds indexes into [runtimeMetricsExporter.samples]
+// for the runtime/metrics equivalents of the runtime.MemStats fields
+// that varz has exported as memstats_* since before runtime/metrics
+// existed. Those names are kept for existing dashboards, but they are
+// now read from runtime/metrics rather than runtime.ReadMemStats,
+// which stops the world.
+type memstatsIndexes struct {
+	heapAlloc  int // /memory/classes/heap/objects:bytes
+	totalAlloc int // /gc/heap/allocs:bytes
+	sys        int // /memory/classes/total:bytes
+	mallocs    int // /gc/heap/allocs:objects
+	frees      int // /gc/heap/frees:objects
+	tinyAllocs int // /gc/heap/tiny/allocs:objects
+	numGC      int // /gc/cycles/total:gc-cycles
+	gcCPU      int // /cpu/classes/gc/total:cpu-seconds
+	gcIdleCPU  int // /cpu/classes/gc/mark/idle:cpu-seconds
+	totalCPU   int // /cpu/classes/total:cpu-seconds
+}
+
+// runtimeMetrics is the process-wide exporter used by [Handler].
+var runtimeMetrics = new(runtimeMetricsExporter)
+
+// runtimeMetricsBufPool holds *[]byte output buffers so that a scrape
+// can be formatted under the exporter's lock and written to the
+// (possibly slow) client after releasing it.
+var runtimeMetricsBufPool = sync.Pool{New: func() any { return new([]byte) }}
+
+// init reads every metric in [runtimeMetricSpecs] once, drops the ones
+// this toolchain does not know, and precomputes the per-metric output
+// prefixes and histogram layouts.
+//
+// e.mu must be held.
+func (e *runtimeMetricsExporter) init() {
+	e.inited = true
+
+	samples := make([]metrics.Sample, len(runtimeMetricSpecs))
+	for i, spec := range runtimeMetricSpecs {
+		samples[i].Name = spec.name
+	}
+	metrics.Read(samples)
+
+	byName := map[string]int{}
+	for i, spec := range runtimeMetricSpecs {
+		s := samples[i]
+		var ok bool
+		switch s.Value.Kind() {
+		case metrics.KindUint64, metrics.KindFloat64:
+			ok = spec.typ != "histogram"
+		case metrics.KindFloat64Histogram:
+			ok = spec.typ == "histogram"
+		}
+		if !ok {
+			// Unknown to this toolchain, or its kind changed
+			// out from under our table.
+			continue
+		}
+		st := &runtimeMetricState{spec: spec}
+		if !spec.hidden {
+			name := runtimeMetricPromName(spec.name, spec.typ)
+			st.typeLine = []byte("# TYPE " + name + " " + spec.typ + "\n")
+			if spec.typ == "histogram" {
+				st.hist = newRuntimeHistLayout(name, s.Value.Float64Histogram().Buckets, spec.buckets)
+				st.countPrefix = []byte(name + "_count ")
+			} else {
+				st.valuePrefix = []byte(name + " ")
+			}
+		}
+		byName[spec.name] = len(e.samples)
+		e.samples = append(e.samples, s)
+		e.states = append(e.states, st)
+	}
+
+	idx := func(name string) int {
+		if i, ok := byName[name]; ok {
+			return i
+		}
+		return -1
+	}
+	e.memstatsIdx = memstatsIndexes{
+		heapAlloc:  idx("/memory/classes/heap/objects:bytes"),
+		totalAlloc: idx("/gc/heap/allocs:bytes"),
+		sys:        idx("/memory/classes/total:bytes"),
+		mallocs:    idx("/gc/heap/allocs:objects"),
+		frees:      idx("/gc/heap/frees:objects"),
+		tinyAllocs: idx("/gc/heap/tiny/allocs:objects"),
+		numGC:      idx("/gc/cycles/total:gc-cycles"),
+		gcCPU:      idx("/cpu/classes/gc/total:cpu-seconds"),
+		gcIdleCPU:  idx("/cpu/classes/gc/mark/idle:cpu-seconds"),
+		totalCPU:   idx("/cpu/classes/total:cpu-seconds"),
+	}
+}
+
+// writeTo reads the current values of the exported metrics and writes
+// them to w in Prometheus text format, preceded by the legacy
+// memstats_* metrics.
+func (e *runtimeMetricsExporter) writeTo(w io.Writer) error {
+	bp := runtimeMetricsBufPool.Get().(*[]byte)
+	defer runtimeMetricsBufPool.Put(bp)
+
+	e.mu.Lock()
+	if !e.inited {
+		e.init()
+	} else {
+		metrics.Read(e.samples)
+	}
+	*bp = e.appendMemstats((*bp)[:0])
+	for i, st := range e.states {
+		if !st.spec.hidden {
+			*bp = st.appendTo(*bp, &e.samples[i].Value)
+		}
+	}
+	e.mu.Unlock()
+
+	_, err := w.Write(*bp)
+	return err
+}
+
+// appendTo appends the Prometheus lines for st with value v to b.
+func (st *runtimeMetricState) appendTo(b []byte, v *metrics.Value) []byte {
+	b = append(b, st.typeLine...)
+	switch v.Kind() {
+	case metrics.KindUint64:
+		b = append(b, st.valuePrefix...)
+		b = strconv.AppendUint(b, v.Uint64(), 10)
+		b = append(b, '\n')
+	case metrics.KindFloat64:
+		b = append(b, st.valuePrefix...)
+		b = strconv.AppendFloat(b, v.Float64(), 'g', -1, 64)
+		b = append(b, '\n')
+	case metrics.KindFloat64Histogram:
+		counts := v.Float64Histogram().Counts
+		var sum uint64
+		var next int
+		for i, end := range st.hist.ends {
+			for ; next < end && next < len(counts); next++ {
+				sum += counts[next]
+			}
+			b = append(b, st.hist.lines[i]...)
+			b = strconv.AppendUint(b, sum, 10)
+			b = append(b, '\n')
+		}
+		b = append(b, st.countPrefix...)
+		b = strconv.AppendUint(b, sum, 10)
+		b = append(b, '\n')
+	}
+	return b
+}
+
+// appendMemstats appends the legacy memstats_* metrics to b, sourced
+// from the runtime/metrics samples named by e.memstatsIdx.
+//
+// e.mu must be held and e.samples must have been read.
+func (e *runtimeMetricsExporter) appendMemstats(b []byte) []byte {
+	idx := &e.memstatsIdx
+	u := func(i int) (uint64, bool) {
+		if i < 0 {
+			return 0, false
+		}
+		v := &e.samples[i].Value
+		if v.Kind() != metrics.KindUint64 {
+			return 0, false
+		}
+		return v.Uint64(), true
+	}
+	f := func(i int) (float64, bool) {
+		if i < 0 {
+			return 0, false
+		}
+		v := &e.samples[i].Value
+		if v.Kind() != metrics.KindFloat64 {
+			return 0, false
+		}
+		return v.Float64(), true
+	}
+	appendUint := func(typ, name string, v uint64, ok bool, help string) {
+		if ok {
+			b = appendMemstat(b, typ, name, help)
+			b = strconv.AppendUint(b, v, 10)
+			b = append(b, '\n')
+		}
+	}
+	appendIdx := func(typ, name string, i int, help string) {
+		v, ok := u(i)
+		appendUint(typ, name, v, ok, help)
+	}
+	appendIdx("gauge", "heap_alloc", idx.heapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
+	appendIdx("counter", "total_alloc", idx.totalAlloc, "cumulative bytes allocated for heap objects")
+	appendIdx("gauge", "sys", idx.sys, "total bytes of memory obtained from the OS")
+
+	// runtime.ReadMemStats counts tiny allocations in both Mallocs
+	// and Frees, but /gc/heap/allocs:objects and /gc/heap/frees:objects
+	// exclude them, so add them back.
+	mallocs, ok1 := u(idx.mallocs)
+	frees, ok2 := u(idx.frees)
+	tiny, ok3 := u(idx.tinyAllocs)
+	appendUint("counter", "mallocs", mallocs+tiny, ok1 && ok3, "cumulative count of heap objects allocated")
+	appendUint("counter", "frees", frees+tiny, ok2 && ok3, "cumulative count of heap objects freed")
+
+	appendIdx("counter", "num_gc", idx.numGC, "number of completed GC cycles")
+	gc, ok1 := f(idx.gcCPU)
+	idle, ok2 := f(idx.gcIdleCPU)
+	total, ok3 := f(idx.totalCPU)
+	if ok1 && ok2 && ok3 {
+		// This is how the runtime computes MemStats.GCCPUFraction:
+		// GC time less idle-priority marking, which is "free", over
+		// total time. The runtime updates its CPU class stats at
+		// the end of each GC cycle, so total is 0 until the first
+		// one, as GCCPUFraction was too.
+		var frac float64
+		if total > 0 {
+			frac = (gc - idle) / total
+		}
+		b = appendMemstat(b, "gauge", "gc_cpu_fraction", "fraction of CPU time used by GC")
+		b = strconv.AppendFloat(b, frac, 'f', -1, 64)
+		b = append(b, '\n')
+	}
+	return b
+}
+
+// appendMemstat appends the HELP and TYPE lines for the memstats_*
+// metric name to b, followed by the metric name and a space, leaving
+// the caller to append the value and newline.
+func appendMemstat(b []byte, typ, name, help string) []byte {
+	b = append(b, "# HELP memstats_"...)
+	b = append(b, name...)
+	b = append(b, ' ')
+	b = append(b, help...)
+	b = append(b, "\n# TYPE memstats_"...)
+	b = append(b, name...)
+	b = append(b, ' ')
+	b = append(b, typ...)
+	b = append(b, "\nmemstats_"...)
+	b = append(b, name...)
+	b = append(b, ' ')
+	return b
+}

tsweb/varz/runtimemetrics_test.go

@@ -0,0 +1,480 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package varz
+
+import (
+	"bytes"
+	"expvar"
+	"io"
+	"math"
+	"net/http/httptest"
+	"runtime"
+	"runtime/metrics"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+	"tailscale.com/util/racebuild"
+	"tailscale.com/version"
+)
+
+// runtimeMetricDescs returns the runtime's metric descriptions by name.
+func runtimeMetricDescs() map[string]metrics.Description {
+	m := map[string]metrics.Description{}
+	for _, d := range metrics.All() {
+		m[d.Name] = d
+	}
+	return m
+}
+
+// TestRuntimeMetricSpecs checks runtimeMetricSpecs against the
+// runtime's own metric descriptions: every spec names a real metric
+// (or a Tailscale fork metric when not built with the fork), its
+// declared Prometheus type matches the metric's kind and
+// cumulativeness, and, when built with the fork, every /tailscale/
+// metric the fork defines is exported.
+func TestRuntimeMetricSpecs(t *testing.T) {
+	descs := runtimeMetricDescs()
+	seen := map[string]bool{}
+	for _, spec := range runtimeMetricSpecs {
+		if seen[spec.name] {
+			t.Errorf("duplicate spec for %q", spec.name)
+		}
+		seen[spec.name] = true
+
+		d, ok := descs[spec.name]
+		if !ok {
+			if strings.HasPrefix(spec.name, "/tailscale/") && !version.IsTailscaleGo() {
+				continue
+			}
+			t.Errorf("spec %q is not a metric in this toolchain", spec.name)
+			continue
+		}
+		var wantTyp string
+		switch {
+		case d.Kind == metrics.KindFloat64Histogram:
+			wantTyp = "histogram"
+		case d.Cumulative:
+			wantTyp = "counter"
+		default:
+			wantTyp = "gauge"
+		}
+		if spec.typ != wantTyp {
+			t.Errorf("spec %q has type %q; runtime description implies %q", spec.name, spec.typ, wantTyp)
+		}
+		if (spec.buckets != 0) != (d.Kind == metrics.KindFloat64Histogram) {
+			t.Errorf("spec %q: buckets=%d but kind=%v; histograms and only histograms need a bucket layout", spec.name, spec.buckets, d.Kind)
+		}
+	}
+
+	if version.IsTailscaleGo() {
+		for name := range descs {
+			if strings.HasPrefix(name, "/tailscale/") && !seen[name] {
+				t.Errorf("Tailscale Go fork metric %q is not in runtimeMetricSpecs", name)
+			}
+		}
+	}
+}
+
+func TestRuntimeMetricPromName(t *testing.T) {
+	tests := []struct {
+		name, typ string
+		want      string
+	}{
+		{"/gc/heap/allocs:bytes", "counter", "go_runtime_gc_heap_allocs_bytes_total"},
+		{"/gc/heap/live:bytes", "gauge", "go_runtime_gc_heap_live_bytes"},
+		{"/cpu/classes/gc/total:cpu-seconds", "counter", "go_runtime_cpu_classes_gc_total_cpu_seconds_total"},
+		{"/sched/goroutines-created:goroutines", "counter", "go_runtime_sched_goroutines_created_goroutines_total"},
+		{"/sched/latencies:seconds", "histogram", "go_runtime_sched_latencies_seconds"},
+		{"/tailscale/sched/goroutines-by-stack-size:bytes", "histogram", "go_runtime_tailscale_sched_goroutines_by_stack_size_bytes"},
+		{"/tailscale/sched/timers/zombies/func:timers", "gauge", "go_runtime_tailscale_sched_timers_zombies_func_timers"},
+		{"/gc/heap/allocs:bytes/second", "gauge", "go_runtime_gc_heap_allocs_bytes_per_second"},
+	}
+	for _, tt := range tests {
+		if got := runtimeMetricPromName(tt.name, tt.typ); got != tt.want {
+			t.Errorf("runtimeMetricPromName(%q, %q) = %q; want %q", tt.name, tt.typ, got, tt.want)
+		}
+	}
+}
+
+// histLayoutLabels returns the le labels of lay, in order.
+func histLayoutLabels(lay *runtimeHistLayout) []string {
+	var out []string
+	for _, line := range lay.lines {
+		s := string(line)
+		_, rest, _ := strings.Cut(s, `{le="`)
+		le, _, _ := strings.Cut(rest, `"`)
+		out = append(out, le)
+	}
+	return out
+}
+
+func TestRuntimeHistLayout(t *testing.T) {
+	t.Run("time", func(t *testing.T) {
+		// Use the runtime's real time histogram buckets, so this
+		// test also notices if the runtime changes its layout.
+		s := []metrics.Sample{{Name: "/sched/latencies:seconds"}}
+		metrics.Read(s)
+		if s[0].Value.Kind() != metrics.KindFloat64Histogram {
+			t.Fatalf("unexpected kind %v", s[0].Value.Kind())
+		}
+		buckets := s[0].Value.Float64Histogram().Buckets
+		t.Logf("runtime has %d time histogram buckets", len(buckets)-1)
+
+		lay := newRuntimeHistLayout("m", buckets, histTime)
+		want := []string{
+			"2.56e-07", "1.024e-06", "4.096e-06", "1.6384e-05", "6.5536e-05",
+			"0.000262144", "0.001048576", "0.004194304", "0.016777216",
+			"0.067108864", "0.268435456", "1.073741824", "+Inf",
+		}
+		if got := histLayoutLabels(lay); !slices.Equal(got, want) {
+			t.Errorf("labels = %q; want %q", got, want)
+		}
+		if got, want := lay.ends[len(lay.ends)-1], len(buckets)-1; got != want {
+			t.Errorf("+Inf bucket covers %d runtime buckets; want all %d", got, want)
+		}
+		if !slices.IsSorted(lay.ends) {
+			t.Errorf("ends not sorted: %v", lay.ends)
+		}
+		// Each merged boundary must be an actual runtime boundary,
+		// so no runtime bucket is split.
+		for i, end := range lay.ends[:len(lay.ends)-1] {
+			le := want[i]
+			if got := formatLE(buckets[end]); got != le {
+				t.Errorf("output bucket %d ends at runtime boundary %v; label says %v", i, got, le)
+			}
+		}
+	})
+
+	t.Run("ints", func(t *testing.T) {
+		// The fork's func zombie lifetime histogram: 0, 1, then
+		// powers of two, then +Inf, counting whole GC cycles.
+		buckets := []float64{0, 1, 2, 4, 8, math.Inf(1)}
+		lay := newRuntimeHistLayout("m", buckets, histInts)
+		want := []string{"0", "1", "3", "7", "+Inf"}
+		if got := histLayoutLabels(lay); !slices.Equal(got, want) {
+			t.Errorf("labels = %q; want %q", got, want)
+		}
+		if want := []int{1, 2, 3, 4, 5}; !slices.Equal(lay.ends, want) {
+			t.Errorf("ends = %v; want %v", lay.ends, want)
+		}
+
+		st := &runtimeMetricState{hist: lay, countPrefix: []byte("m_count ")}
+		var v metrics.Value
+		setHistValue(t, &v, buckets, []uint64{10, 3, 2, 1, 4})
+		got := string(st.appendTo(nil, &v))
+		const wantOut = "m_bucket{le=\"0\"} 10\n" +
+			"m_bucket{le=\"1\"} 13\n" +
+			"m_bucket{le=\"3\"} 15\n" +
+			"m_bucket{le=\"7\"} 16\n" +
+			"m_bucket{le=\"+Inf\"} 20\n" +
+			"m_count 20\n"
+		if got != wantOut {
+			t.Errorf("got:\n%s\nwant:\n%s", got, wantOut)
+		}
+	})
+
+	t.Run("powers_of_two", func(t *testing.T) {
+		buckets := []float64{2048, 4096, 8192, 16384, math.Inf(1)}
+		lay := newRuntimeHistLayout("m", buckets, histPowersOfTwo)
+		want := []string{"2048", "4096", "8192", "+Inf"}
+		if got := histLayoutLabels(lay); !slices.Equal(got, want) {
+			t.Errorf("labels = %q; want %q", got, want)
+		}
+		if want := []int{1, 2, 3, 4}; !slices.Equal(lay.ends, want) {
+			t.Errorf("ends = %v; want %v", lay.ends, want)
+		}
+
+		// A value exactly at a lower bound lands in the bucket
+		// labeled with that value.
+		st := &runtimeMetricState{hist: lay, countPrefix: []byte("m_count ")}
+		var v metrics.Value
+		setHistValue(t, &v, buckets, []uint64{5, 0, 2, 1})
+		got := string(st.appendTo(nil, &v))
+		const wantOut = "m_bucket{le=\"2048\"} 5\n" +
+			"m_bucket{le=\"4096\"} 5\n" +
+			"m_bucket{le=\"8192\"} 7\n" +
+			"m_bucket{le=\"+Inf\"} 8\n" +
+			"m_count 8\n"
+		if got != wantOut {
+			t.Errorf("got:\n%s\nwant:\n%s", got, wantOut)
+		}
+	})
+}
+
+// formatLE formats f the way runtime histogram le labels are formatted.
+func formatLE(f float64) string {
+	if math.IsInf(f, 1) {
+		return "+Inf"
+	}
+	return string(strconv.AppendFloat(nil, f, 'g', -1, 64))
+}
+
+// setHistValue makes v a histogram value with the given buckets and
+// counts, by reading a real histogram metric and overwriting its
+// storage, since runtime/metrics offers no constructor.
+func setHistValue(t *testing.T, v *metrics.Value, buckets []float64, counts []uint64) {
+	t.Helper()
+	s := []metrics.Sample{{Name: "/sched/latencies:seconds"}}
+	metrics.Read(s)
+	h := s[0].Value.Float64Histogram()
+	if h == nil {
+		t.Fatal("no histogram")
+	}
+	h.Buckets = buckets
+	h.Counts = counts
+	*v = s[0].Value
+}
+
+// parseProm parses Prometheus text format, failing the test on error.
+func parseProm(t *testing.T, b []byte) map[string]*dto.MetricFamily {
+	t.Helper()
+	p := expfmt.NewTextParser(model.LegacyValidation)
+	fams, err := p.TextToMetricFamilies(bytes.NewReader(b))
+	if err != nil {
+		t.Fatalf("parsing Prometheus output: %v\n%s", err, b)
+	}
+	return fams
+}
+
+func TestRuntimeMetricsExporter(t *testing.T) {
+	var buf bytes.Buffer
+	if err := runtimeMetrics.writeTo(&buf); err != nil {
+		t.Fatal(err)
+	}
+	out := buf.Bytes()
+	t.Logf("output:\n%s", out)
+	fams := parseProm(t, out)
+
+	// Every spec that this toolchain supports must appear with the
+	// right type.
+	descs := runtimeMetricDescs()
+	for _, spec := range runtimeMetricSpecs {
+		if _, ok := descs[spec.name]; !ok {
+			continue
+		}
+		if spec.hidden {
+			if _, ok := fams[runtimeMetricPromName(spec.name, spec.typ)]; ok {
+				t.Errorf("hidden spec %q was exported", spec.name)
+			}
+			continue
+		}
+		name := runtimeMetricPromName(spec.name, spec.typ)
+		fam, ok := fams[name]
+		if !ok {
+			t.Errorf("no output for %q (%s)", spec.name, name)
+			continue
+		}
+		var wantTyp dto.MetricType
+		switch spec.typ {
+		case "counter":
+			wantTyp = dto.MetricType_COUNTER
+		case "gauge":
+			wantTyp = dto.MetricType_GAUGE
+		case "histogram":
+			wantTyp = dto.MetricType_HISTOGRAM
+		}
+		if fam.GetType() != wantTyp {
+			t.Errorf("%s: type %v; want %v", name, fam.GetType(), wantTyp)
+		}
+		if spec.typ == "histogram" {
+			h := fam.GetMetric()[0].GetHistogram()
+			if len(h.GetBucket()) == 0 {
+				t.Errorf("%s: no buckets", name)
+			}
+			var prev uint64
+			for _, b := range h.GetBucket() {
+				if b.GetCumulativeCount() < prev {
+					t.Errorf("%s: bucket counts not cumulative", name)
+				}
+				prev = b.GetCumulativeCount()
+			}
+			if h.GetSampleCount() != prev {
+				t.Errorf("%s: count %d != last bucket %d", name, h.GetSampleCount(), prev)
+			}
+		}
+	}
+
+	// The legacy memstats_* names must still be exported with their
+	// old types and plausible values.
+	memstats := []struct {
+		name string
+		typ  dto.MetricType
+	}{
+		{"memstats_heap_alloc", dto.MetricType_GAUGE},
+		{"memstats_total_alloc", dto.MetricType_COUNTER},
+		{"memstats_sys", dto.MetricType_GAUGE},
+		{"memstats_mallocs", dto.MetricType_COUNTER},
+		{"memstats_frees", dto.MetricType_COUNTER},
+		{"memstats_num_gc", dto.MetricType_COUNTER},
+		{"memstats_gc_cpu_fraction", dto.MetricType_GAUGE},
+	}
+	for _, ms := range memstats {
+		fam, ok := fams[ms.name]
+		if !ok {
+			t.Errorf("missing %s", ms.name)
+			continue
+		}
+		if fam.GetType() != ms.typ {
+			t.Errorf("%s: type %v; want %v", ms.name, fam.GetType(), ms.typ)
+		}
+		if fam.GetHelp() == "" {
+			t.Errorf("%s: no HELP", ms.name)
+		}
+	}
+	if v := fams["memstats_sys"].GetMetric()[0].GetGauge().GetValue(); v <= 0 {
+		t.Errorf("memstats_sys = %v; want > 0", v)
+	}
+	if v := fams["memstats_gc_cpu_fraction"].GetMetric()[0].GetGauge().GetValue(); v < 0 || v > 1 {
+		t.Errorf("memstats_gc_cpu_fraction = %v; want in [0, 1]", v)
+	}
+
+	if version.IsTailscaleGo() {
+		fam, ok := fams["go_runtime_tailscale_sched_goroutines_by_stack_size_bytes"]
+		if !ok {
+			t.Fatal("missing stack size histogram under tailscale_go")
+		}
+		h := fam.GetMetric()[0].GetHistogram()
+		if h.GetSampleCount() == 0 {
+			t.Error("stack size histogram counts no goroutines")
+		}
+		// The first bucket is the minimum stack size, 2 KiB on most
+		// platforms but larger on some, and always a power of two.
+		if got := h.GetBucket()[0].GetUpperBound(); got < 2048 || math.Log2(got) != math.Trunc(math.Log2(got)) {
+			t.Errorf("first stack size bucket is %v; want a power of two >= 2048", got)
+		}
+	} else {
+		for name := range fams {
+			if strings.Contains(name, "tailscale") {
+				t.Errorf("unexpected fork metric %q without tailscale_go", name)
+			}
+		}
+	}
+}
+
+// TestVarzHandlerRuntimeMetrics checks that Handler includes the
+// runtime metrics and skips the expvar package's stop-the-world
+// memstats var, and that the whole output parses.
+func TestVarzHandlerRuntimeMetrics(t *testing.T) {
+	rec := httptest.NewRecorder()
+	Handler(rec, httptest.NewRequest("GET", "/", nil))
+	out := rec.Body.Bytes()
+	fams := parseProm(t, out)
+	for _, name := range []string{
+		"go_runtime_gc_heap_live_bytes",
+		"go_runtime_cpu_classes_gc_total_cpu_seconds_total",
+		"go_runtime_sched_latencies_seconds",
+		"memstats_heap_alloc",
+	} {
+		if _, ok := fams[name]; !ok {
+			t.Errorf("missing %s", name)
+		}
+	}
+	if n := bytes.Count(out, []byte("\nmemstats_heap_alloc ")); n != 1 {
+		t.Errorf("memstats_heap_alloc appears %d times; want 1", n)
+	}
+	if bytes.Contains(out, []byte(`skipping expvar "memstats"`)) {
+		t.Errorf("output mentions skipping memstats; it should not be reached at all")
+	}
+}
+
+// TestMemstatsMatchReadMemStats checks that the memstats_* values
+// computed from runtime/metrics agree with the runtime.MemStats fields
+// they replaced, read back to back in the same process.
+func TestMemstatsMatchReadMemStats(t *testing.T) {
+	// Make sure at least one GC has happened so the CPU stats and
+	// GCCPUFraction are populated, then read both as close together
+	// as possible. The exporter's first call allocates, so warm it.
+	runtime.GC()
+	runtimeMetrics.writeTo(io.Discard)
+	var buf bytes.Buffer
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	runtimeMetrics.writeTo(&buf)
+	fams := parseProm(t, buf.Bytes())
+
+	get := func(name string) float64 {
+		t.Helper()
+		fam, ok := fams[name]
+		if !ok {
+			t.Fatalf("missing %s", name)
+		}
+		m := fam.GetMetric()[0]
+		if fam.GetType() == dto.MetricType_COUNTER {
+			return m.GetCounter().GetValue()
+		}
+		return m.GetGauge().GetValue()
+	}
+	// Anything allocated between the two reads shows up as a small
+	// difference, so allow a little slack on the heap numbers. The GC
+	// cycle count and the GC CPU fraction only change at GC, which
+	// runtime.GC just did, so they should match closely.
+	check := func(name string, want, relTol, absTol float64) {
+		t.Helper()
+		got := get(name)
+		diff := math.Abs(got - want)
+		if diff > absTol && diff > relTol*math.Abs(want) {
+			t.Errorf("%s = %v; runtime.MemStats says %v", name, got, want)
+		}
+	}
+	check("memstats_heap_alloc", float64(ms.HeapAlloc), 0, 64<<10)
+	check("memstats_total_alloc", float64(ms.TotalAlloc), 0, 64<<10)
+	check("memstats_sys", float64(ms.Sys), 0.01, 0)
+	check("memstats_mallocs", float64(ms.Mallocs), 0, 50)
+	check("memstats_frees", float64(ms.Frees), 0, 50)
+	check("memstats_num_gc", float64(ms.NumGC), 0, 1)
+	check("memstats_gc_cpu_fraction", ms.GCCPUFraction, 0, 1e-6)
+	t.Logf("gc_cpu_fraction: runtime/metrics=%v MemStats=%v", get("memstats_gc_cpu_fraction"), ms.GCCPUFraction)
+}
+
+// TestWritePrometheusExpvarSkipsMemstats checks that callers walking
+// expvar.Do themselves (as some corp code does) also never trigger the
+// stop-the-world memstats func.
+func TestWritePrometheusExpvarSkipsMemstats(t *testing.T) {
+	v := expvar.Get("memstats")
+	if v == nil {
+		t.Fatal("expvar package did not publish memstats")
+	}
+	var buf bytes.Buffer
+	WritePrometheusExpvar(&buf, expvar.KeyValue{Key: "memstats", Value: v})
+	if buf.Len() != 0 {
+		t.Errorf("got output for memstats expvar; want none:\n%s", buf.String())
+	}
+}
+
+func TestRuntimeMetricsExporterAllocs(t *testing.T) {
+	if racebuild.On {
+		t.Skip("allocation counts differ under the race detector")
+	}
+	e := new(runtimeMetricsExporter)
+	e.writeTo(io.Discard) // init
+	e.writeTo(io.Discard) // grow the pooled buffer
+	if allocs := testing.AllocsPerRun(100, func() {
+		e.writeTo(io.Discard)
+	}); allocs != 0 {
+		t.Errorf("allocs per scrape = %v; want 0", allocs)
+	}
+}
+
+func BenchmarkRuntimeMetricsExporter(b *testing.B) {
+	e := new(runtimeMetricsExporter)
+	e.writeTo(io.Discard)
+	b.ReportAllocs()
+	for b.Loop() {
+		e.writeTo(io.Discard)
+	}
+}
+
+func BenchmarkVarzHandler(b *testing.B) {
+	req := httptest.NewRequest("GET", "/", nil)
+	rec := new(httptest.ResponseRecorder) // no body, so writes are discarded
+	b.ReportAllocs()
+	for b.Loop() {
+		Handler(rec, req)
+	}
+}

tsweb/varz/varz.go

@@ -5,7 +5,6 @@
 package varz
 
 import (
-	"bufio"
 	"cmp"
 	"expvar"
 	"fmt"
@@ -23,10 +22,8 @@ import (
 	"unicode"
 	"unicode/utf8"
 
-	"golang.org/x/exp/constraints"
 	"tailscale.com/metrics"
 	"tailscale.com/syncs"
-	"tailscale.com/types/logger"
 	"tailscale.com/version"
 )
 
@@ -215,11 +212,14 @@ func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) {
 	if typ == "" {
 		var funcRet string
 		if f, ok := kv.Value.(expvar.Func); ok {
-			v := f()
-			if ms, ok := v.(runtime.MemStats); ok && name == "memstats" {
-				writeMemstats(w, &ms)
+			if key == "memstats" {
+				// The expvar package's own memstats var calls
+				// runtime.ReadMemStats, which stops the world and
+				// allocates. [Handler] exports the memstats_* metrics
+				// from runtime/metrics instead, so don't even call it.
 				return
 			}
+			v := f()
 			if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") {
 				if name == "version" {
 					fmt.Fprintf(w, "%s{version=%q,binary=%q} 1\n", name, vs, binaryName())
@@ -328,13 +328,19 @@ type sortedKVs struct {
 //     is not exported.
 //
 // This will evolve over time, or perhaps be replaced.
+//
+// It also exports the Go runtime/metrics in [runtimeMetricSpecs] and
+// the legacy memstats_* metrics, both read from runtime/metrics.
 func Handler(w http.ResponseWriter, r *http.Request) {
 	ExpvarDoHandler(expvarDo)(w, r)
+	runtimeMetrics.writeTo(w)
 }
 
 // ExpvarDoHandler handler returns a Handler like above, but takes an optional
 // expvar.Do func allow the usage of alternative containers of metrics, other
 // than the global expvar.Map.
+//
+// Unlike [Handler], it does not export the Go runtime/metrics.
 func ExpvarDoHandler(expvarDoFunc func(f func(expvar.KeyValue))) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/plain;version=0.0.4;charset=utf-8")
@@ -378,54 +384,6 @@ type PrometheusMetricsReflectRooter interface {
 
 var expvarDo = expvar.Do // pulled out for tests
 
-func writeMemstat[V constraints.Integer | constraints.Float](bw *bufio.Writer, typ, name string, v V, help string) {
-	if help != "" {
-		bw.WriteString("# HELP memstats_")
-		bw.WriteString(name)
-		bw.WriteString(" ")
-		bw.WriteString(help)
-		bw.WriteByte('\n')
-	}
-	bw.WriteString("# TYPE memstats_")
-	bw.WriteString(name)
-	bw.WriteString(" ")
-	bw.WriteString(typ)
-	bw.WriteByte('\n')
-	bw.WriteString("memstats_")
-	bw.WriteString(name)
-	bw.WriteByte(' ')
-	rt := reflect.TypeOf(v)
-	switch {
-	case rt == reflect.TypeFor[int]() ||
-		rt == reflect.TypeFor[uint]() ||
-		rt == reflect.TypeFor[int8]() ||
-		rt == reflect.TypeFor[uint8]() ||
-		rt == reflect.TypeFor[int16]() ||
-		rt == reflect.TypeFor[uint16]() ||
-		rt == reflect.TypeFor[int32]() ||
-		rt == reflect.TypeFor[uint32]() ||
-		rt == reflect.TypeFor[int64]() ||
-		rt == reflect.TypeFor[uint64]() ||
-		rt == reflect.TypeFor[uintptr]():
-		bw.Write(strconv.AppendInt(bw.AvailableBuffer(), int64(v), 10))
-	case rt == reflect.TypeFor[float32]() || rt == reflect.TypeFor[float64]():
-		bw.Write(strconv.AppendFloat(bw.AvailableBuffer(), float64(v), 'f', -1, 64))
-	}
-	bw.WriteByte('\n')
-}
-
-func writeMemstats(w io.Writer, ms *runtime.MemStats) {
-	fmt.Fprintf(w, "%v", logger.ArgWriter(func(bw *bufio.Writer) {
-		writeMemstat(bw, "gauge", "heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
-		writeMemstat(bw, "counter", "total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects")
-		writeMemstat(bw, "gauge", "sys", ms.Sys, "total bytes of memory obtained from the OS")
-		writeMemstat(bw, "counter", "mallocs", ms.Mallocs, "cumulative count of heap objects allocated")
-		writeMemstat(bw, "counter", "frees", ms.Frees, "cumulative count of heap objects freed")
-		writeMemstat(bw, "counter", "num_gc", ms.NumGC, "number of completed GC cycles")
-		writeMemstat(bw, "gauge", "gc_cpu_fraction", ms.GCCPUFraction, "fraction of CPU time used by GC")
-	}))
-}
-
 // sortedStructField is metadata about a struct field used both for sorting once
 // (by structTypeSortedFields) and at serving time (by
 // foreachExportedStructField).

tsweb/varz/varz_test.go

@@ -4,18 +4,15 @@
 package varz
 
 import (
-	"bytes"
 	"expvar"
 	"net/http/httptest"
 	"reflect"
-	"runtime"
 	"strings"
 	"testing"
 
 	"tailscale.com/metrics"
 	"tailscale.com/syncs"
 	"tailscale.com/tstest"
-	"tailscale.com/util/racebuild"
 	"tailscale.com/version"
 )
 
@@ -340,7 +337,7 @@ api_status_code 42
 				f(expvar.KeyValue{Key: tt.k, Value: tt.v})
 			})
 			rec := httptest.NewRecorder()
-			Handler(rec, httptest.NewRequest("GET", "/", nil))
+			ExpvarDoHandler(expvarDo)(rec, httptest.NewRequest("GET", "/", nil))
 			if got := rec.Body.Bytes(); string(got) != tt.want {
 				t.Errorf("mismatch\n got: %q\n%s\nwant: %q\n%s\n", got, got, tt.want, tt.want)
 			}
@@ -453,7 +450,7 @@ func TestVarzHandlerSorting(t *testing.T) {
 	})
 	rec := httptest.NewRecorder()
 	req := httptest.NewRequest("GET", "/", nil)
-	Handler(rec, req)
+	ExpvarDoHandler(expvarDo)(rec, req)
 	got := rec.Body.Bytes()
 	const want = "# TYPE aa gauge\naa 0\n# TYPE zz counter\nzz 0\n"
 	if string(got) != want {
@@ -471,75 +468,3 @@ func TestVarzHandlerSorting(t *testing.T) {
 		}
 	}
 }
-
-func TestWriteMemestats(t *testing.T) {
-	memstats := &runtime.MemStats{
-		Alloc:        1,
-		TotalAlloc:   2,
-		Sys:          3,
-		Lookups:      4,
-		Mallocs:      5,
-		Frees:        6,
-		HeapAlloc:    7,
-		HeapSys:      8,
-		HeapIdle:     9,
-		HeapInuse:    10,
-		HeapReleased: 11,
-		HeapObjects:  12,
-		StackInuse:   13,
-		StackSys:     14,
-		MSpanInuse:   15,
-		MSpanSys:     16,
-		MCacheInuse:  17,
-		MCacheSys:    18,
-		BuckHashSys:  19,
-		GCSys:        20,
-		OtherSys:     21,
-		NextGC:       22,
-		LastGC:       23,
-		PauseTotalNs: 24,
-		// PauseNs:       [256]int64{},
-		NumGC:         26,
-		NumForcedGC:   27,
-		GCCPUFraction: 0.28,
-	}
-
-	var buf bytes.Buffer
-	writeMemstats(&buf, memstats)
-	lines := strings.Split(buf.String(), "\n")
-
-	checkFor := func(name, typ, value string) {
-		var foundType, foundValue bool
-		for _, line := range lines {
-			if line == "memstats_"+name+" "+value {
-				foundValue = true
-			}
-			if line == "# TYPE memstats_"+name+" "+typ {
-				foundType = true
-			}
-			if foundValue && foundType {
-				return
-			}
-		}
-		t.Errorf("memstats_%s foundType=%v foundValue=%v", name, foundType, foundValue)
-	}
-
-	t.Logf("memstats:\n %s", buf.String())
-
-	checkFor("heap_alloc", "gauge", "7")
-	checkFor("total_alloc", "counter", "2")
-	checkFor("sys", "gauge", "3")
-	checkFor("mallocs", "counter", "5")
-	checkFor("frees", "counter", "6")
-	checkFor("num_gc", "counter", "26")
-	checkFor("gc_cpu_fraction", "gauge", "0.28")
-
-	if !racebuild.On {
-		if allocs := testing.AllocsPerRun(1000, func() {
-			buf.Reset()
-			writeMemstats(&buf, memstats)
-		}); allocs != 1 {
-			t.Errorf("allocs = %v; want max %v", allocs, 1)
-		}
-	}
-}