mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-21 20:22:41 +09:00
feature/conn25: shut the address assignment queue down from the sender side (#22269)
extension.Shutdown closed client.addrsCh, but the hooks that feed that channel are still live at that point. LocalBackend.shutdown shuts the extension host down before closing the engine, and nothing unregisters what installHooks registered on the DNS manager and the tun device, so a DNS response arriving during the shutdown window still reaches mapDNSResponse -> rewriteDNSResponse -> reserveAddresses -> enqueueAddressAssignment, and sends on the closed channel. That's a panic, not a dropped assignment. Every send already happens under c.mu, so the client can shut its own queue down: client.shutdown takes the lock, sets a closed flag and drains the queue, and enqueueAddressAssignment refuses with errClientShutdown from then on. A late DNS response now fails the reservation instead of queueing it for a sendLoop that is gone. reserveAddresses rolls back, returning the magic and transit IPs to their pools, and the query servfails rather than being answered with a magic IP that no connector will ever hear about. Fixes https://github.com/tailscale/corp/issues/51235 Change-Id: I712ba8493e77b8176e133decc6097efb52b1b4ab Signed-off-by: Mazdak Nasab <mazdak.nasab@gmail.com>
This commit is contained in:
1 parent
1e2c689b6b
commit
6372b9a43a
3 files changed
+99
-2
No files matched your search
@@ -32,7 +32,8 @@ type client struct {
|
||||
|
||||
// Remember to add new fields to [client.reset] if needed.
|
||||
mu sync.Mutex // protects the fields below
|
||||
generation uint64 // Current generation ID
|
||||
closed bool
|
||||
generation uint64 // Current generation ID
|
||||
v4MagicIPPool *ippool
|
||||
v4TransitIPPool *ippool
|
||||
v6MagicIPPool *ippool
|
||||
@@ -103,6 +104,23 @@ func (c *client) reset() {
|
||||
// Drop any assignments still waiting to be sent. They describe the state
|
||||
// we just cleared, so delivering them would register transit IPs with
|
||||
// connectors that this client no longer knows anything about.
|
||||
c.drainAddrsLocked()
|
||||
}
|
||||
|
||||
// shutdown stops the client from accepting further address assignments and
|
||||
// drops any that are still queued.
|
||||
func (c *client) shutdown() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.closed = true
|
||||
c.drainAddrsLocked()
|
||||
close(c.addrsCh)
|
||||
}
|
||||
|
||||
// drainAddrsLocked empties the send queue. c.mu must be held, which is what
|
||||
// makes the drain complete: senders can't refill the queue behind it.
|
||||
func (c *client) drainAddrsLocked() {
|
||||
for {
|
||||
select {
|
||||
case _, ok := <-c.addrsCh:
|
||||
@@ -241,7 +259,16 @@ func (c *client) addTransitIPForConnector(as addrs) error {
|
||||
// the assignment was dropped. It's backpressure, not a hard failure.
|
||||
var errQueueFull = errors.New("queue full")
|
||||
|
||||
// errClientShutdown is returned once [client.shutdown] has run and the client
|
||||
// no longer accepts address assignments.
|
||||
var errClientShutdown = errors.New("conn25 client shut down")
|
||||
|
||||
// enqueueAddressAssignment queues addrs to be registered with its connector by
|
||||
// sendLoop. c.mu must be held.
|
||||
func (c *client) enqueueAddressAssignment(addrs *addrs) error {
|
||||
if c.closed {
|
||||
return errClientShutdown
|
||||
}
|
||||
select {
|
||||
// TODO(fran) investigate the value of waiting for multiple addresses and sending them
|
||||
// in one ConnectorTransitIPRequest
|
||||
|
||||
@@ -365,7 +365,7 @@ func (e *extension) Shutdown() error {
|
||||
e.ctxCancel(errors.New("extension shutdown"))
|
||||
}
|
||||
if e.conn25 != nil {
|
||||
close(e.conn25.client.addrsCh)
|
||||
e.conn25.client.shutdown()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ package conn25
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
@@ -1884,6 +1885,75 @@ func newTestSafeBackend(t *testing.T) *testSafeBackend {
|
||||
|
||||
func (b *testSafeBackend) Sys() *tsd.System { return b.sys }
|
||||
|
||||
// TestShutdownDropsLateDNSResponse covers the shutdown ordering in
|
||||
// [ipnlocal.LocalBackend]: the extension host is shut down before the engine is
|
||||
// closed, and nothing unregisters the hooks that installHooks set on the DNS
|
||||
// manager and the tun device. So a DNS response can still reach mapDNSResponse
|
||||
// after Shutdown has returned, and that path sends on client.addrsCh. The
|
||||
// assignment has to be refused rather than queued for a sendLoop that is gone,
|
||||
// and refusing it must not panic the way closing the channel would.
|
||||
func TestShutdownDropsLateDNSResponse(t *testing.T) {
|
||||
ext := &extension{
|
||||
conn25: newConn25(logger.Discard),
|
||||
backend: newTestSafeBackend(t),
|
||||
}
|
||||
if err := ext.Init(&testHost{
|
||||
nb: &testNodeBackend{peers: []tailcfg.NodeView{testConnectorPeer}},
|
||||
prefs: testPrefsNotConnector,
|
||||
authReconfigAsync: func() {},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ext.conn25.reconfig(mustConfig(t, makeSelfNode(t, []appctype.AppConnectorAttr{{
|
||||
Name: testAppName,
|
||||
Connectors: []string{"tag:woo"},
|
||||
Domains: []string{"example.com"},
|
||||
Mode: appctype.MagicIP,
|
||||
}}, arbitraryPools, nil)))
|
||||
|
||||
if err := ext.Shutdown(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A DNS response that the resolver was already handling when the extension
|
||||
// shut down. This panics with "send on closed channel" if Shutdown closes
|
||||
// the send queue instead of shutting it down from the sending side.
|
||||
out := ext.mapDNSResponse(makeDNSResponse(t, "example.com.", []*dnsmessage.AResource{{A: [4]byte{1, 0, 0, 0}}}))
|
||||
|
||||
// The query can't be answered with a magic IP that no connector will ever
|
||||
// hear about, so it servfails.
|
||||
var p dnsmessage.Parser
|
||||
hdr, err := p.Start(out)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing mapped response: %v", err)
|
||||
}
|
||||
if hdr.RCode != dnsmessage.RCodeServerFailure {
|
||||
t.Errorf("RCode after shutdown: got %v, want %v", hdr.RCode, dnsmessage.RCodeServerFailure)
|
||||
}
|
||||
|
||||
c := ext.conn25.client
|
||||
if got := len(c.assignments.byMagicIP); got != 0 {
|
||||
t.Errorf("want no assignments after shutdown, got %d", got)
|
||||
}
|
||||
if got := len(c.addrsCh); got != 0 {
|
||||
t.Errorf("want nothing queued after shutdown, got %d queued assignments", got)
|
||||
}
|
||||
// The refused reservation has to roll back, not leak its addresses.
|
||||
if got := c.v4MagicIPPool.inUseCount(); got != 0 {
|
||||
t.Errorf("magic pool has %d addresses in use after shutdown, want 0", got)
|
||||
}
|
||||
if got := c.v4TransitIPPool.inUseCount(); got != 0 {
|
||||
t.Errorf("transit pool has %d addresses in use after shutdown, want 0", got)
|
||||
}
|
||||
|
||||
// The profile state change hook is still registered too, and must not
|
||||
// bring the client back to life.
|
||||
c.reset()
|
||||
if _, err := c.reserveAddresses(testAppName, "example.com.", netip.MustParseAddr("1.0.0.0"), 10, testConnectorPeer, ""); !errors.Is(err, errClientShutdown) {
|
||||
t.Errorf("reserveAddresses after reset: got %v, want errClientShutdown", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAddressAssignmentIsHandled tests that after enqueueAddress has been called
|
||||
// we handle the assignment asynchronously by:
|
||||
// - making a peerapi request to a peer.
|
||||
|
||||
Reference in new issue
Block a user