net/dnscache, feature/dnsresolvecache: persist DNS resolutions to disk

Add a new modular dnsresolvecache feature that records every successful
DNS resolution from net/dnscache as a JSON file per hostname in
$statedir/dns-cache/, so a later boot with misconfigured DNS can still
find last-known-good IPs for critical hostnames like the control plane.
Files are rewritten only when their contents change, so a file's
modification time records when the answer last changed.

When regular DNS resolution fails, the disk cache is now consulted
before the DERP-based bootstrap DNS in net/dnsfallback. This is the
first step toward removing the DERP-based mechanism: new clientmetrics
(dnscache_disk_fallback_hit, dnscache_disk_fallback_miss,
dnscache_derp_fallback_ok, dnscache_derp_fallback_dial_ok) will tell us
when the DERP path no longer fires in the fleet and can be deleted.

The feature is linked into tailscaled by default (omittable with
ts_omit_dnsresolvecache) and is not included in tsnet.

Updates #21028

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
Change-Id: I59228cbf68e3b48dfb1215cdd12bd8166ab14034

cmd/tailscaled/depaware.txt

@@ -309,6 +309,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/feature/condregister/useproxy                  from tailscale.com/feature/condregister
         tailscale.com/feature/conn25                                 from tailscale.com/feature/condregister
         tailscale.com/feature/debugportmapper                        from tailscale.com/feature/condregister
+        tailscale.com/feature/dnsresolvecache                        from tailscale.com/feature/condregister
         tailscale.com/feature/doctor                                 from tailscale.com/feature/condregister
         tailscale.com/feature/drive                                  from tailscale.com/feature/condregister
         tailscale.com/feature/favorites                              from tailscale.com/feature/condregister

cmd/tailscaled/deps_test.go

@@ -72,6 +72,17 @@ func TestOmitSyslog(t *testing.T) {
 	}.Check(t)
 }
 
+func TestOmitDNSResolveCache(t *testing.T) {
+	deptest.DepChecker{
+		GOOS:   "linux",
+		GOARCH: "amd64",
+		Tags:   "ts_omit_dnsresolvecache,ts_include_cli",
+		BadDeps: map[string]string{
+			"tailscale.com/feature/dnsresolvecache": "unexpected dnsresolvecache usage with ts_omit_dnsresolvecache",
+		},
+	}.Check(t)
+}
+
 func TestOmitSyspolicy(t *testing.T) {
 	const msg = "unexpected syspolicy usage with ts_omit_syspolicy"
 	deptest.DepChecker{

cmd/tailscaled/tailscaled.go

@@ -45,6 +45,7 @@ import (
 	"tailscale.com/logpolicy"
 	"tailscale.com/logtail"
 	"tailscale.com/net/dns"
+	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/netns"
@@ -726,6 +727,9 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 	}
 	if root := lb.TailscaleVarRoot(); root != "" {
 		dnsfallback.SetCachePath(filepath.Join(root, "derpmap.cached.json"), logf)
+		if f, ok := dnscache.HookSetCacheDir.GetOk(); ok {
+			f(filepath.Join(root, "dns-cache"), logf)
+		}
 	}
 	if f, ok := hookConfigureWebClient.GetOk(); ok {
 		f(lb)

feature/buildfeatures/feature_dnsresolvecache_disabled.go

@@ -0,0 +1,13 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Code generated by gen.go; DO NOT EDIT.
+
+//go:build ts_omit_dnsresolvecache
+
+package buildfeatures
+
+// HasDNSResolveCache is whether the binary was built with support for modular feature "Persist successful DNS resolutions to disk for use on later boots with broken DNS".
+// Specifically, it's whether the binary was NOT built with the "ts_omit_dnsresolvecache" build tag.
+// It's a const so it can be used for dead code elimination.
+const HasDNSResolveCache = false

feature/buildfeatures/feature_dnsresolvecache_enabled.go

@@ -0,0 +1,13 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Code generated by gen.go; DO NOT EDIT.
+
+//go:build !ts_omit_dnsresolvecache
+
+package buildfeatures
+
+// HasDNSResolveCache is whether the binary was built with support for modular feature "Persist successful DNS resolutions to disk for use on later boots with broken DNS".
+// Specifically, it's whether the binary was NOT built with the "ts_omit_dnsresolvecache" build tag.
+// It's a const so it can be used for dead code elimination.
+const HasDNSResolveCache = true

feature/condregister/maybe_dnsresolvecache.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !ts_omit_dnsresolvecache
+
+package condregister
+
+import _ "tailscale.com/feature/dnsresolvecache"

feature/dnsresolvecache/dnsresolvecache.go

@@ -0,0 +1,195 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package dnsresolvecache persists successful DNS resolutions from
+// net/dnscache to disk, one JSON file per hostname, so that a later
+// tailscaled boot with misconfigured DNS can still find
+// last-known-good IPs for critical hostnames such as the control
+// plane. It is intended to eventually replace the DERP-based
+// bootstrap DNS in net/dnsfallback.
+//
+// A file is rewritten only when its contents change, so its
+// modification time records when the answer last changed, not when
+// it was last confirmed.
+//
+// This package is linked into tailscaled by default and omitted from
+// tsnet. Nothing here is automatic for tsnet-based apps: to use it,
+// they must both blank-import this package and configure the cache
+// directory themselves by invoking [dnscache.HookSetCacheDir], which
+// is otherwise only called by tailscaled at startup.
+//
+// This package's state is process-global. In tsnet-based apps running
+// multiple tsnet.Server instances in one process, only the cache
+// directory from the first [dnscache.HookSetCacheDir] call is used;
+// later calls are ignored and all servers share the first cache.
+package dnsresolvecache
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+
+	"tailscale.com/atomicfile"
+	"tailscale.com/feature"
+	"tailscale.com/net/dnscache"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/mak"
+)
+
+func init() {
+	feature.Register("dnsresolvecache")
+	dnscache.HookSetCacheDir.Set(setCacheDir)
+	dnscache.HookPersistResolution.Set(persist)
+	dnscache.HookLookupDiskCache.Set(lookup)
+}
+
+var (
+	mu          sync.Mutex        // guards the variables below
+	cacheDir    string            // empty until the first successful setCacheDir call
+	logf        logger.Logf       // non-nil once cacheDir is set
+	lastWritten map[string]string // hostname => digest of last JSON written
+)
+
+// record is the JSON structure of each persisted per-hostname file.
+type record struct {
+	Resolver string       // resolver source of the answer: "forward", "cloud", or "fallback"
+	A        []netip.Addr `json:",omitempty"` // IPv4 addresses, sorted
+	AAAA     []netip.Addr `json:",omitempty"` // IPv6 addresses, sorted
+}
+
+// setCacheDir creates dir if needed and enables persistence to it.
+// It implements [dnscache.HookSetCacheDir].
+//
+// Only the first successful call has any effect; later calls are
+// no-ops that keep using the first directory. See the package doc
+// for what that means for multi-server tsnet apps.
+func setCacheDir(dir string, lf logger.Logf) {
+	if lf == nil {
+		lf = logger.Discard
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if cacheDir != "" {
+		if dir != cacheDir {
+			lf("dnsresolvecache: cache dir already set to %q; ignoring %q", cacheDir, dir)
+		}
+		return
+	}
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		lf("dnsresolvecache: %v", err)
+		return
+	}
+	cacheDir = dir
+	logf = lf
+}
+
+// filePath returns the path of the cache file for host.
+// The caller must have validated host and hold mu (for cacheDir).
+func filePath(host string) string {
+	return filepath.Join(cacheDir, "dns-"+host+".json")
+}
+
+// persist writes the resolution of host to disk, unless the on-disk
+// contents would be unchanged. It implements
+// [dnscache.HookPersistResolution].
+func persist(host, resolver string, ips []netip.Addr) {
+	host = strings.ToLower(host)
+	if !validHostname(host) || len(ips) == 0 {
+		return
+	}
+	rec := record{Resolver: resolver}
+	ips = slices.Clone(ips)
+	slices.SortFunc(ips, netip.Addr.Compare)
+	for _, ip := range slices.Compact(ips) {
+		if ip.Is4() {
+			rec.A = append(rec.A, ip)
+		} else {
+			rec.AAAA = append(rec.AAAA, ip)
+		}
+	}
+	j, err := json.Marshal(rec)
+	if err != nil {
+		return
+	}
+	digest := sha256.Sum256(j)
+
+	mu.Lock()
+	defer mu.Unlock()
+	if cacheDir == "" {
+		return
+	}
+	path := filePath(host)
+	if last, ok := lastWritten[host]; ok {
+		if last == string(digest[:]) {
+			return
+		}
+	} else if old, err := os.ReadFile(path); err == nil && bytes.Equal(old, j) {
+		// First resolution since process start and the file already
+		// matches; skip the write to preserve its modtime.
+		mak.Set(&lastWritten, host, string(digest[:]))
+		return
+	}
+	if err := atomicfile.WriteFile(path, j, 0600); err != nil {
+		logf("dnsresolvecache: writing %v: %v", path, err)
+		return
+	}
+	mak.Set(&lastWritten, host, string(digest[:]))
+}
+
+// lookup returns the persisted last-known-good IPs for host, if any.
+// It implements [dnscache.HookLookupDiskCache].
+func lookup(host string) ([]netip.Addr, bool) {
+	host = strings.ToLower(host)
+	if !validHostname(host) {
+		return nil, false
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if cacheDir == "" {
+		return nil, false
+	}
+	j, err := os.ReadFile(filePath(host))
+	if err != nil {
+		return nil, false
+	}
+	var rec record
+	if err := json.Unmarshal(j, &rec); err != nil {
+		logf("dnsresolvecache: parsing cache for %q: %v", host, err)
+		return nil, false
+	}
+	ips := append(rec.A, rec.AAAA...)
+	if len(ips) == 0 {
+		return nil, false
+	}
+	return ips, true
+}
+
+// validHostname reports whether host is a DNS hostname that is safe
+// to embed in a filename: dot-separated non-empty labels of
+// lowercase letters, digits, hyphens, and underscores. It is
+// intentionally stricter than DNS itself (which permits nearly
+// arbitrary bytes in labels) to keep hostile names out of paths.
+func validHostname(host string) bool {
+	if len(host) == 0 || len(host) > 253 {
+		return false
+	}
+	for label := range strings.SplitSeq(host, ".") {
+		if len(label) == 0 || len(label) > 63 {
+			return false
+		}
+		for i := range len(label) {
+			b := label[i]
+			if b >= 'a' && b <= 'z' || b >= '0' && b <= '9' || b == '-' || b == '_' {
+				continue
+			}
+			return false
+		}
+	}
+	return true
+}

feature/dnsresolvecache/dnsresolvecache_test.go

@@ -0,0 +1,177 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package dnsresolvecache
+
+import (
+	"net/netip"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+)
+
+func setDirForTest(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	mu.Lock()
+	cacheDir = dir
+	logf = t.Logf
+	lastWritten = nil
+	mu.Unlock()
+	t.Cleanup(func() {
+		mu.Lock()
+		cacheDir = ""
+		logf = nil
+		lastWritten = nil
+		mu.Unlock()
+	})
+	return dir
+}
+
+func mustIPs(ss ...string) (ips []netip.Addr) {
+	for _, s := range ss {
+		ips = append(ips, netip.MustParseAddr(s))
+	}
+	return ips
+}
+
+func mtime(t *testing.T, path string) time.Time {
+	t.Helper()
+	fi, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return fi.ModTime()
+}
+
+func TestPersistAndLookup(t *testing.T) {
+	dir := setDirForTest(t)
+
+	persist("Ctrl.Example.COM", "forward", mustIPs("4.4.4.4", "2600::2", "1.1.1.1", "2600::1", "1.1.1.1"))
+
+	path := filepath.Join(dir, "dns-ctrl.example.com.json")
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	const want = `{"Resolver":"forward","A":["1.1.1.1","4.4.4.4"],"AAAA":["2600::1","2600::2"]}`
+	if string(got) != want {
+		t.Errorf("file contents = %q; want %q", got, want)
+	}
+
+	ips, ok := lookup("ctrl.example.com")
+	if !ok {
+		t.Fatal("lookup failed")
+	}
+	if want := mustIPs("1.1.1.1", "4.4.4.4", "2600::1", "2600::2"); !slices.Equal(ips, want) {
+		t.Errorf("lookup = %v; want %v", ips, want)
+	}
+
+	if _, ok := lookup("other.example.com"); ok {
+		t.Error("lookup of unknown host unexpectedly succeeded")
+	}
+}
+
+func TestNoRewriteWhenUnchanged(t *testing.T) {
+	dir := setDirForTest(t)
+	path := filepath.Join(dir, "dns-ctrl.example.com.json")
+	old := time.Now().Add(-time.Hour).Round(time.Second)
+
+	persist("ctrl.example.com", "forward", mustIPs("1.1.1.1"))
+	if err := os.Chtimes(path, old, old); err != nil {
+		t.Fatal(err)
+	}
+
+	persist("ctrl.example.com", "forward", mustIPs("1.1.1.1"))
+	if got := mtime(t, path); !got.Equal(old) {
+		t.Errorf("unchanged persist rewrote file; mtime = %v, want %v", got, old)
+	}
+
+	// A fresh process (no in-memory digests) should also not rewrite
+	// an identical file.
+	mu.Lock()
+	lastWritten = nil
+	mu.Unlock()
+	persist("ctrl.example.com", "forward", mustIPs("1.1.1.1"))
+	if got := mtime(t, path); !got.Equal(old) {
+		t.Errorf("unchanged persist after restart rewrote file; mtime = %v, want %v", got, old)
+	}
+
+	persist("ctrl.example.com", "forward", mustIPs("2.2.2.2"))
+	if got := mtime(t, path); got.Equal(old) {
+		t.Error("changed persist did not rewrite file")
+	}
+	if ips, ok := lookup("ctrl.example.com"); !ok || !slices.Equal(ips, mustIPs("2.2.2.2")) {
+		t.Errorf("lookup after change = %v, %v; want [2.2.2.2], true", ips, ok)
+	}
+}
+
+func TestInvalidHostnames(t *testing.T) {
+	dir := setDirForTest(t)
+	for _, host := range []string{
+		"",
+		".",
+		"..",
+		"../evil",
+		"foo/bar.example.com",
+		"foo..example.com",
+		".example.com",
+		"example.com.", // trailing dot: valid DNS, but not a name we store
+		"bad*char.example.com",
+	} {
+		persist(host, "forward", mustIPs("1.1.1.1"))
+		if _, ok := lookup(host); ok {
+			t.Errorf("lookup(%q) unexpectedly succeeded", host)
+		}
+	}
+	des, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, de := range des {
+		t.Errorf("unexpected file %q written for invalid hostname", de.Name())
+	}
+}
+
+func TestLookupCorruptFile(t *testing.T) {
+	dir := setDirForTest(t)
+	if err := os.WriteFile(filepath.Join(dir, "dns-ctrl.example.com.json"), []byte("not json"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := lookup("ctrl.example.com"); ok {
+		t.Error("lookup of corrupt file unexpectedly succeeded")
+	}
+}
+
+func TestSetCacheDirFirstCallWins(t *testing.T) {
+	t.Cleanup(func() {
+		mu.Lock()
+		cacheDir = ""
+		logf = nil
+		lastWritten = nil
+		mu.Unlock()
+	})
+	dir1 := t.TempDir()
+	dir2 := t.TempDir()
+	setCacheDir(dir1, t.Logf)
+
+	// A second call is a no-op; the first directory stays in use.
+	setCacheDir(dir2, t.Logf)
+	persist("ctrl.example.com", "forward", mustIPs("1.1.1.1"))
+	if _, err := os.Stat(filepath.Join(dir1, "dns-ctrl.example.com.json")); err != nil {
+		t.Errorf("file not written to first dir: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir2, "dns-ctrl.example.com.json")); err == nil {
+		t.Error("file unexpectedly written to second dir")
+	}
+}
+
+func TestDisabled(t *testing.T) {
+	// No setCacheDir call: both directions are no-ops.
+	persist("ctrl.example.com", "forward", mustIPs("1.1.1.1"))
+	if _, ok := lookup("ctrl.example.com"); ok {
+		t.Error("lookup unexpectedly succeeded with no cache dir set")
+	}
+}

feature/featuretags/featuretags.go

@@ -164,6 +164,7 @@ var Features = map[FeatureTag]FeatureMeta{
 		Deps: []FeatureTag{"portmapper"},
 	},
 	"desktop_sessions": {Sym: "DesktopSessions", Desc: "Desktop sessions support"},
+	"dnsresolvecache":  {Sym: "DNSResolveCache", Desc: "Persist successful DNS resolutions to disk for use on later boots with broken DNS"},
 	"doctor":           {Sym: "Doctor", Desc: "Diagnose possible issues with Tailscale and its host environment"},
 	"drive":            {Sym: "Drive", Desc: "Tailscale Drive (file server) support"},
 	"flashappliance":   {Sym: "FlashAppliance", Desc: "'tailscale configure flash-appliance' and 'pve-appliance' CLI commands for deploying Tailscale appliance images"},

net/dnscache/dnscache.go

@@ -20,14 +20,51 @@ import (
 	"time"
 
 	"tailscale.com/envknob"
+	"tailscale.com/feature"
+	"tailscale.com/feature/buildfeatures"
 	"tailscale.com/net/netx"
 	"tailscale.com/syncs"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/cloudenv"
 	"tailscale.com/util/singleflight"
 	"tailscale.com/util/testenv"
 )
 
+// HookSetCacheDir optionally points to the dnsresolvecache feature's
+// function to configure the directory in which resolved IPs are
+// persisted to disk. It is called by tailscaled at startup, if the
+// feature is linked in.
+var HookSetCacheDir feature.Hook[func(dir string, logf logger.Logf)]
+
+// HookPersistResolution optionally points to the dnsresolvecache
+// feature's function to record a successful DNS resolution of host
+// to disk. The resolver argument is one of the "forward", "cloud",
+// or "fallback" resolver source names.
+var HookPersistResolution feature.Hook[func(host, resolver string, ips []netip.Addr)]
+
+// HookLookupDiskCache optionally points to the dnsresolvecache
+// feature's function to load previously persisted IPs for host from
+// disk. It is consulted only after regular DNS resolution has
+// failed, before falling back to the DERP-based bootstrap DNS.
+var HookLookupDiskCache feature.Hook[func(host string) ([]netip.Addr, bool)]
+
+// Resolver source names, as passed to [HookPersistResolution] and
+// used for deciding fallback behavior in Resolver.lookupIP.
+const (
+	srcForward  = "forward"  // Resolver.Forward (or the test hook)
+	srcCloud    = "cloud"    // cloud host resolver (e.g. GCP metadata resolver)
+	srcDisk     = "disk"     // dnsresolvecache disk cache of an earlier resolution
+	srcFallback = "fallback" // Resolver.LookupIPFallback (DERP-based bootstrap DNS)
+)
+
+var (
+	metricDiskFallbackHit    = clientmetric.NewCounter("dnscache_disk_fallback_hit")
+	metricDiskFallbackMiss   = clientmetric.NewCounter("dnscache_disk_fallback_miss")
+	metricDERPFallbackOK     = clientmetric.NewCounter("dnscache_derp_fallback_ok")
+	metricDERPFallbackDialOK = clientmetric.NewCounter("dnscache_derp_fallback_dial_ok")
+)
+
 var zaddr netip.Addr
 
 var single = &Resolver{
@@ -295,15 +332,28 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (ip, ip6 netip.Add
 	defer lookupCancel()
 
 	var ips []netip.Addr
+	src := srcForward
 	if r.LookupIPForTest != nil && testenv.InTest() {
 		ips, err = r.LookupIPForTest(ctx, host)
 	} else {
 		ips, err = r.fwd().LookupNetIP(lookupCtx, "ip", host)
+		if err != nil || len(ips) == 0 {
+			if resolver, ok := r.cloudHostResolver(); ok {
+				r.dlogf("resolving %q via cloud resolver", host)
+				src = srcCloud
+				ips, err = resolver.LookupNetIP(lookupCtx, "ip", host)
+			}
+		}
 	}
-	if err != nil || len(ips) == 0 {
-		if resolver, ok := r.cloudHostResolver(); ok {
-			r.dlogf("resolving %q via cloud resolver", host)
-			ips, err = resolver.LookupNetIP(lookupCtx, "ip", host)
+	if buildfeatures.HasDNSResolveCache && (err != nil || len(ips) == 0) {
+		if f, ok := HookLookupDiskCache.GetOk(); ok {
+			if cached, ok := f(host); ok {
+				r.dlogf("resolving %q from disk cache after error", host)
+				metricDiskFallbackHit.Add(1)
+				ips, err, src = cached, nil, srcDisk
+			} else {
+				metricDiskFallbackMiss.Add(1)
+			}
 		}
 	}
 	if (err != nil || len(ips) == 0) && r.LookupIPFallback != nil {
@@ -314,7 +364,11 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (ip, ip6 netip.Add
 		} else {
 			r.dlogf("resolving %q using fallback resolver due to no returned IPs", host)
 		}
+		src = srcFallback
 		ips, err = r.LookupIPFallback(lookupCtx, host)
+		if err == nil && len(ips) > 0 {
+			metricDERPFallbackOK.Add(1)
+		}
 	}
 	if err != nil {
 		return netip.Addr{}, netip.Addr{}, nil, err
@@ -344,11 +398,11 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (ip, ip6 netip.Add
 			}
 		}
 	}
-	r.addIPCache(host, ip, ip6, ips, r.ttl())
+	r.addIPCache(host, src, ip, ip6, ips, r.ttl())
 	return ip, ip6, ips, nil
 }
 
-func (r *Resolver) addIPCache(host string, ip, ip6 netip.Addr, allIPs []netip.Addr, d time.Duration) {
+func (r *Resolver) addIPCache(host, src string, ip, ip6 netip.Addr, allIPs []netip.Addr, d time.Duration) {
 	if ip.IsPrivate() {
 		// Don't cache obviously wrong entries from captive portals.
 		// TODO: use DoH or DoT for the forwarding resolver?
@@ -356,6 +410,14 @@ func (r *Resolver) addIPCache(host string, ip, ip6 netip.Addr, allIPs []netip.Ad
 		return
 	}
 
+	// Skip persisting disk-sourced results to avoid rewriting the
+	// disk cache files with their own contents.
+	if buildfeatures.HasDNSResolveCache && src != srcDisk {
+		if f, ok := HookPersistResolution.GetOk(); ok {
+			f(host, src, allIPs)
+		}
+	}
+
 	r.dlogf("%q resolved to IP %v; caching", host, ip)
 
 	r.mu.Lock()
@@ -416,6 +478,7 @@ func (d *dialer) DialContext(ctx context.Context, network, address string) (retC
 			return
 		}
 		if c, err := dc.raceDial(ctx, ips); err == nil {
+			metricDERPFallbackDialOK.Add(1)
 			retConn = c
 			ret = nil
 			return

net/dnscache/dnscache_test.go

@@ -298,3 +298,113 @@ func TestSingleHostStaticResult(t *testing.T) {
 		})
 	}
 }
+
+type persistCall struct {
+	host, resolver string
+	ips            []netip.Addr
+}
+
+func TestDiskCacheHooks(t *testing.T) {
+	mustIPs := func(ss ...string) (ips []netip.Addr) {
+		for _, s := range ss {
+			ips = append(ips, netip.MustParseAddr(s))
+		}
+		return ips
+	}
+	errFailed := errors.New("some resolution failure")
+
+	t.Run("persist-on-success", func(t *testing.T) {
+		var calls []persistCall
+		defer HookPersistResolution.SetForTest(func(host, resolver string, ips []netip.Addr) {
+			calls = append(calls, persistCall{host, resolver, ips})
+		})()
+		r := &Resolver{
+			Logf: t.Logf,
+			LookupIPForTest: func(ctx context.Context, host string) ([]netip.Addr, error) {
+				return mustIPs("1.1.1.1", "2600::1"), nil
+			},
+		}
+		if _, _, _, err := r.LookupIP(t.Context(), "ctrl.example.com"); err != nil {
+			t.Fatal(err)
+		}
+		want := []persistCall{{"ctrl.example.com", "forward", mustIPs("1.1.1.1", "2600::1")}}
+		if !reflect.DeepEqual(calls, want) {
+			t.Errorf("persist calls = %+v; want %+v", calls, want)
+		}
+	})
+
+	t.Run("disk-hit-before-derp", func(t *testing.T) {
+		defer HookLookupDiskCache.SetForTest(func(host string) ([]netip.Addr, bool) {
+			if host != "ctrl.example.com" {
+				t.Errorf("disk lookup host = %q; want ctrl.example.com", host)
+			}
+			return mustIPs("2.2.2.2"), true
+		})()
+		var persisted []persistCall
+		defer HookPersistResolution.SetForTest(func(host, resolver string, ips []netip.Addr) {
+			persisted = append(persisted, persistCall{host, resolver, ips})
+		})()
+		r := &Resolver{
+			Logf: t.Logf,
+			LookupIPForTest: func(ctx context.Context, host string) ([]netip.Addr, error) {
+				return nil, errFailed
+			},
+			LookupIPFallback: func(ctx context.Context, host string) ([]netip.Addr, error) {
+				t.Error("DERP fallback used despite disk cache hit")
+				return nil, errFailed
+			},
+		}
+		hits0 := metricDiskFallbackHit.Value()
+		ip, _, _, err := r.LookupIP(t.Context(), "ctrl.example.com")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if want := netip.MustParseAddr("2.2.2.2"); ip != want {
+			t.Errorf("ip = %v; want %v", ip, want)
+		}
+		if d := metricDiskFallbackHit.Value() - hits0; d != 1 {
+			t.Errorf("disk fallback hit metric delta = %d; want 1", d)
+		}
+		if len(persisted) != 0 {
+			t.Errorf("disk-sourced result was re-persisted: %+v", persisted)
+		}
+	})
+
+	t.Run("disk-miss-uses-derp", func(t *testing.T) {
+		defer HookLookupDiskCache.SetForTest(func(host string) ([]netip.Addr, bool) {
+			return nil, false
+		})()
+		var persisted []persistCall
+		defer HookPersistResolution.SetForTest(func(host, resolver string, ips []netip.Addr) {
+			persisted = append(persisted, persistCall{host, resolver, ips})
+		})()
+		r := &Resolver{
+			Logf: t.Logf,
+			LookupIPForTest: func(ctx context.Context, host string) ([]netip.Addr, error) {
+				return nil, errFailed
+			},
+			LookupIPFallback: func(ctx context.Context, host string) ([]netip.Addr, error) {
+				return mustIPs("3.3.3.3"), nil
+			},
+		}
+		miss0 := metricDiskFallbackMiss.Value()
+		derp0 := metricDERPFallbackOK.Value()
+		ip, _, _, err := r.LookupIP(t.Context(), "ctrl.example.com")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if want := netip.MustParseAddr("3.3.3.3"); ip != want {
+			t.Errorf("ip = %v; want %v", ip, want)
+		}
+		if d := metricDiskFallbackMiss.Value() - miss0; d != 1 {
+			t.Errorf("disk fallback miss metric delta = %d; want 1", d)
+		}
+		if d := metricDERPFallbackOK.Value() - derp0; d != 1 {
+			t.Errorf("DERP fallback ok metric delta = %d; want 1", d)
+		}
+		want := []persistCall{{"ctrl.example.com", "fallback", mustIPs("3.3.3.3")}}
+		if !reflect.DeepEqual(persisted, want) {
+			t.Errorf("persist calls = %+v; want %+v", persisted, want)
+		}
+	})
+}

tsnet/tsnet_test.go

@@ -3555,6 +3555,7 @@ func TestDeps(t *testing.T) {
 			"tailscale.com/feature/bird":                    "tsnet should not depend on BIRD integration",
 			"tailscale.com/feature/captiveportal":           "tsnet apps don't need captive portal detection; import it explicitly if desired",
 			"tailscale.com/feature/clientupdate":            "tsnet should not depend on feature/clientupdate",
+			"tailscale.com/feature/dnsresolvecache":         "tsnet apps don't persist DNS resolutions to disk by default; import it explicitly if desired",
 			"tailscale.com/feature/remoteconfig":            "tsnet should not depend on feature/remoteconfig",
 			"tailscale.com/feature/syspolicy":               "tsnet should not depend on syspolicy",
 			"tailscale.com/ipn/store/awsstore":              "tsnet callers wanting AWS state storage should import awsstore themselves",

tstest/integration/tailscaled_deps_test_darwin.go

@@ -31,6 +31,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/dnscache"
 	_ "tailscale.com/net/dnsfallback"
 	_ "tailscale.com/net/netmon"
 	_ "tailscale.com/net/netns"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -31,6 +31,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/dnscache"
 	_ "tailscale.com/net/dnsfallback"
 	_ "tailscale.com/net/netmon"
 	_ "tailscale.com/net/netns"

tstest/integration/tailscaled_deps_test_linux.go

@@ -31,6 +31,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/dnscache"
 	_ "tailscale.com/net/dnsfallback"
 	_ "tailscale.com/net/netmon"
 	_ "tailscale.com/net/netns"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -31,6 +31,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/dnscache"
 	_ "tailscale.com/net/dnsfallback"
 	_ "tailscale.com/net/netmon"
 	_ "tailscale.com/net/netns"

tstest/integration/tailscaled_deps_test_windows.go

@@ -40,6 +40,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/logtail"
 	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/dnscache"
 	_ "tailscale.com/net/dnsfallback"
 	_ "tailscale.com/net/netmon"
 	_ "tailscale.com/net/netns"