feature/conn25: shut the address assignment queue down from the sender side (#22269)

extension.Shutdown closed client.addrsCh, but the hooks that feed that
channel are still live at that point. LocalBackend.shutdown shuts the
extension host down before closing the engine, and nothing unregisters
what installHooks registered on the DNS manager and the tun device, so a
DNS response arriving during the shutdown window still reaches
mapDNSResponse -> rewriteDNSResponse -> reserveAddresses ->
enqueueAddressAssignment, and sends on the closed channel. That's a
panic, not a dropped assignment.

Every send already happens under c.mu, so the client can shut its own
queue down: client.shutdown takes the lock, sets a closed flag and
drains the queue, and enqueueAddressAssignment refuses with
errClientShutdown from then on.

A late DNS response now fails the reservation instead of queueing it for
a sendLoop that is gone. reserveAddresses rolls back, returning the
magic and transit IPs to their pools, and the query servfails rather
than being answered with a magic IP that no connector will ever hear
about.

Fixes https://github.com/tailscale/corp/issues/51235

Change-Id: I712ba8493e77b8176e133decc6097efb52b1b4ab

Signed-off-by: Mazdak Nasab <mazdak.nasab@gmail.com>
This commit is contained in:
Mazdak Nasab authored and GitHub committed 2026-11-09 13:37:58 -07:00
1 parent 1e2c689b6b
commit 6372b9a43a
3 files changed
+99 -2

No files matched your search

+28 -1
View File
@@ -32,7 +32,8 @@ type client struct {
// Remember to add new fields to [client.reset] if needed.
mu sync.Mutex // protects the fields below
generation uint64 // Current generation ID
closed bool
generation uint64 // Current generation ID
v4MagicIPPool *ippool
v4TransitIPPool *ippool
v6MagicIPPool *ippool
@@ -103,6 +104,23 @@ func (c *client) reset() {
// Drop any assignments still waiting to be sent. They describe the state
// we just cleared, so delivering them would register transit IPs with
// connectors that this client no longer knows anything about.
c.drainAddrsLocked()
}
// shutdown stops the client from accepting further address assignments and
// drops any that are still queued.
func (c *client) shutdown() {
c.mu.Lock()
defer c.mu.Unlock()
c.closed = true
c.drainAddrsLocked()
close(c.addrsCh)
}
// drainAddrsLocked empties the send queue. c.mu must be held, which is what
// makes the drain complete: senders can't refill the queue behind it.
func (c *client) drainAddrsLocked() {
for {
select {
case _, ok := <-c.addrsCh:
@@ -241,7 +259,16 @@ func (c *client) addTransitIPForConnector(as addrs) error {
// the assignment was dropped. It's backpressure, not a hard failure.
var errQueueFull = errors.New("queue full")
// errClientShutdown is returned once [client.shutdown] has run and the client
// no longer accepts address assignments.
var errClientShutdown = errors.New("conn25 client shut down")
// enqueueAddressAssignment queues addrs to be registered with its connector by
// sendLoop. c.mu must be held.
func (c *client) enqueueAddressAssignment(addrs *addrs) error {
if c.closed {
return errClientShutdown
}
select {
// TODO(fran) investigate the value of waiting for multiple addresses and sending them
// in one ConnectorTransitIPRequest
+1 -1
View File
@@ -365,7 +365,7 @@ func (e *extension) Shutdown() error {
e.ctxCancel(errors.New("extension shutdown"))
}
if e.conn25 != nil {
close(e.conn25.client.addrsCh)
e.conn25.client.shutdown()
}
return nil
}
+70
View File
@@ -6,6 +6,7 @@ package conn25
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
@@ -1884,6 +1885,75 @@ func newTestSafeBackend(t *testing.T) *testSafeBackend {
func (b *testSafeBackend) Sys() *tsd.System { return b.sys }
// TestShutdownDropsLateDNSResponse covers the shutdown ordering in
// [ipnlocal.LocalBackend]: the extension host is shut down before the engine is
// closed, and nothing unregisters the hooks that installHooks set on the DNS
// manager and the tun device. So a DNS response can still reach mapDNSResponse
// after Shutdown has returned, and that path sends on client.addrsCh. The
// assignment has to be refused rather than queued for a sendLoop that is gone,
// and refusing it must not panic the way closing the channel would.
func TestShutdownDropsLateDNSResponse(t *testing.T) {
ext := &extension{
conn25: newConn25(logger.Discard),
backend: newTestSafeBackend(t),
}
if err := ext.Init(&testHost{
nb: &testNodeBackend{peers: []tailcfg.NodeView{testConnectorPeer}},
prefs: testPrefsNotConnector,
authReconfigAsync: func() {},
}); err != nil {
t.Fatal(err)
}
ext.conn25.reconfig(mustConfig(t, makeSelfNode(t, []appctype.AppConnectorAttr{{
Name: testAppName,
Connectors: []string{"tag:woo"},
Domains: []string{"example.com"},
Mode: appctype.MagicIP,
}}, arbitraryPools, nil)))
if err := ext.Shutdown(); err != nil {
t.Fatal(err)
}
// A DNS response that the resolver was already handling when the extension
// shut down. This panics with "send on closed channel" if Shutdown closes
// the send queue instead of shutting it down from the sending side.
out := ext.mapDNSResponse(makeDNSResponse(t, "example.com.", []*dnsmessage.AResource{{A: [4]byte{1, 0, 0, 0}}}))
// The query can't be answered with a magic IP that no connector will ever
// hear about, so it servfails.
var p dnsmessage.Parser
hdr, err := p.Start(out)
if err != nil {
t.Fatalf("parsing mapped response: %v", err)
}
if hdr.RCode != dnsmessage.RCodeServerFailure {
t.Errorf("RCode after shutdown: got %v, want %v", hdr.RCode, dnsmessage.RCodeServerFailure)
}
c := ext.conn25.client
if got := len(c.assignments.byMagicIP); got != 0 {
t.Errorf("want no assignments after shutdown, got %d", got)
}
if got := len(c.addrsCh); got != 0 {
t.Errorf("want nothing queued after shutdown, got %d queued assignments", got)
}
// The refused reservation has to roll back, not leak its addresses.
if got := c.v4MagicIPPool.inUseCount(); got != 0 {
t.Errorf("magic pool has %d addresses in use after shutdown, want 0", got)
}
if got := c.v4TransitIPPool.inUseCount(); got != 0 {
t.Errorf("transit pool has %d addresses in use after shutdown, want 0", got)
}
// The profile state change hook is still registered too, and must not
// bring the client back to life.
c.reset()
if _, err := c.reserveAddresses(testAppName, "example.com.", netip.MustParseAddr("1.0.0.0"), 10, testConnectorPeer, ""); !errors.Is(err, errClientShutdown) {
t.Errorf("reserveAddresses after reset: got %v, want errClientShutdown", err)
}
}
// TestAddressAssignmentIsHandled tests that after enqueueAddress has been called
// we handle the assignment asynchronously by:
// - making a peerapi request to a peer.