From 6372b9a43a5cbc3628bf65bd0ec40b4823510d2d Mon Sep 17 00:00:00 2001 From: Mazdak Nasab Date: Mon, 9 Nov 2026 13:37:58 -0700 Subject: [PATCH] feature/conn25: shut the address assignment queue down from the sender side (#22269) extension.Shutdown closed client.addrsCh, but the hooks that feed that channel are still live at that point. LocalBackend.shutdown shuts the extension host down before closing the engine, and nothing unregisters what installHooks registered on the DNS manager and the tun device, so a DNS response arriving during the shutdown window still reaches mapDNSResponse -> rewriteDNSResponse -> reserveAddresses -> enqueueAddressAssignment, and sends on the closed channel. That's a panic, not a dropped assignment. Every send already happens under c.mu, so the client can shut its own queue down: client.shutdown takes the lock, sets a closed flag and drains the queue, and enqueueAddressAssignment refuses with errClientShutdown from then on. A late DNS response now fails the reservation instead of queueing it for a sendLoop that is gone. reserveAddresses rolls back, returning the magic and transit IPs to their pools, and the query servfails rather than being answered with a magic IP that no connector will ever hear about. Fixes https://github.com/tailscale/corp/issues/51235 Change-Id: I712ba8493e77b8176e133decc6097efb52b1b4ab Signed-off-by: Mazdak Nasab --- feature/conn25/client.go | 29 ++++++++++++++- feature/conn25/conn25.go | 2 +- feature/conn25/conn25_test.go | 70 +++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/feature/conn25/client.go b/feature/conn25/client.go index bc9484ec9..6e144ec05 100644 --- a/feature/conn25/client.go +++ b/feature/conn25/client.go @@ -32,7 +32,8 @@ type client struct { // Remember to add new fields to [client.reset] if needed. mu sync.Mutex // protects the fields below - generation uint64 // Current generation ID + closed bool + generation uint64 // Current generation ID v4MagicIPPool *ippool v4TransitIPPool *ippool v6MagicIPPool *ippool @@ -103,6 +104,23 @@ func (c *client) reset() { // Drop any assignments still waiting to be sent. They describe the state // we just cleared, so delivering them would register transit IPs with // connectors that this client no longer knows anything about. + c.drainAddrsLocked() +} + +// shutdown stops the client from accepting further address assignments and +// drops any that are still queued. +func (c *client) shutdown() { + c.mu.Lock() + defer c.mu.Unlock() + + c.closed = true + c.drainAddrsLocked() + close(c.addrsCh) +} + +// drainAddrsLocked empties the send queue. c.mu must be held, which is what +// makes the drain complete: senders can't refill the queue behind it. +func (c *client) drainAddrsLocked() { for { select { case _, ok := <-c.addrsCh: @@ -241,7 +259,16 @@ func (c *client) addTransitIPForConnector(as addrs) error { // the assignment was dropped. It's backpressure, not a hard failure. var errQueueFull = errors.New("queue full") +// errClientShutdown is returned once [client.shutdown] has run and the client +// no longer accepts address assignments. +var errClientShutdown = errors.New("conn25 client shut down") + +// enqueueAddressAssignment queues addrs to be registered with its connector by +// sendLoop. c.mu must be held. func (c *client) enqueueAddressAssignment(addrs *addrs) error { + if c.closed { + return errClientShutdown + } select { // TODO(fran) investigate the value of waiting for multiple addresses and sending them // in one ConnectorTransitIPRequest diff --git a/feature/conn25/conn25.go b/feature/conn25/conn25.go index a6ccc53ec..9b78011fb 100644 --- a/feature/conn25/conn25.go +++ b/feature/conn25/conn25.go @@ -365,7 +365,7 @@ func (e *extension) Shutdown() error { e.ctxCancel(errors.New("extension shutdown")) } if e.conn25 != nil { - close(e.conn25.client.addrsCh) + e.conn25.client.shutdown() } return nil } diff --git a/feature/conn25/conn25_test.go b/feature/conn25/conn25_test.go index a124f2358..3ac07c040 100644 --- a/feature/conn25/conn25_test.go +++ b/feature/conn25/conn25_test.go @@ -6,6 +6,7 @@ package conn25 import ( "context" "encoding/json" + "errors" "fmt" "math" "net/http" @@ -1884,6 +1885,75 @@ func newTestSafeBackend(t *testing.T) *testSafeBackend { func (b *testSafeBackend) Sys() *tsd.System { return b.sys } +// TestShutdownDropsLateDNSResponse covers the shutdown ordering in +// [ipnlocal.LocalBackend]: the extension host is shut down before the engine is +// closed, and nothing unregisters the hooks that installHooks set on the DNS +// manager and the tun device. So a DNS response can still reach mapDNSResponse +// after Shutdown has returned, and that path sends on client.addrsCh. The +// assignment has to be refused rather than queued for a sendLoop that is gone, +// and refusing it must not panic the way closing the channel would. +func TestShutdownDropsLateDNSResponse(t *testing.T) { + ext := &extension{ + conn25: newConn25(logger.Discard), + backend: newTestSafeBackend(t), + } + if err := ext.Init(&testHost{ + nb: &testNodeBackend{peers: []tailcfg.NodeView{testConnectorPeer}}, + prefs: testPrefsNotConnector, + authReconfigAsync: func() {}, + }); err != nil { + t.Fatal(err) + } + ext.conn25.reconfig(mustConfig(t, makeSelfNode(t, []appctype.AppConnectorAttr{{ + Name: testAppName, + Connectors: []string{"tag:woo"}, + Domains: []string{"example.com"}, + Mode: appctype.MagicIP, + }}, arbitraryPools, nil))) + + if err := ext.Shutdown(); err != nil { + t.Fatal(err) + } + + // A DNS response that the resolver was already handling when the extension + // shut down. This panics with "send on closed channel" if Shutdown closes + // the send queue instead of shutting it down from the sending side. + out := ext.mapDNSResponse(makeDNSResponse(t, "example.com.", []*dnsmessage.AResource{{A: [4]byte{1, 0, 0, 0}}})) + + // The query can't be answered with a magic IP that no connector will ever + // hear about, so it servfails. + var p dnsmessage.Parser + hdr, err := p.Start(out) + if err != nil { + t.Fatalf("parsing mapped response: %v", err) + } + if hdr.RCode != dnsmessage.RCodeServerFailure { + t.Errorf("RCode after shutdown: got %v, want %v", hdr.RCode, dnsmessage.RCodeServerFailure) + } + + c := ext.conn25.client + if got := len(c.assignments.byMagicIP); got != 0 { + t.Errorf("want no assignments after shutdown, got %d", got) + } + if got := len(c.addrsCh); got != 0 { + t.Errorf("want nothing queued after shutdown, got %d queued assignments", got) + } + // The refused reservation has to roll back, not leak its addresses. + if got := c.v4MagicIPPool.inUseCount(); got != 0 { + t.Errorf("magic pool has %d addresses in use after shutdown, want 0", got) + } + if got := c.v4TransitIPPool.inUseCount(); got != 0 { + t.Errorf("transit pool has %d addresses in use after shutdown, want 0", got) + } + + // The profile state change hook is still registered too, and must not + // bring the client back to life. + c.reset() + if _, err := c.reserveAddresses(testAppName, "example.com.", netip.MustParseAddr("1.0.0.0"), 10, testConnectorPeer, ""); !errors.Is(err, errClientShutdown) { + t.Errorf("reserveAddresses after reset: got %v, want errClientShutdown", err) + } +} + // TestAddressAssignmentIsHandled tests that after enqueueAddress has been called // we handle the assignment asynchronously by: // - making a peerapi request to a peer.