cmd/containerboot,kube/{certs,health,localclient}: serve /livez and /readyz

Proxies only serve /healthz, which reports whether the node has a tailnet IP,
so it can't back a liveness or readiness probe. When TS_ENABLE_HEALTH_CHECK is
set, containerboot now also serves /livez, which fails only when tailscaled
stops answering LocalAPI, and /readyz, which passes once the node has tailnet
IPs and, with TS_SERVE_CONFIG set, has applied the serve config and has a cert
for each HTTPS-only Tailscale Service it advertises. Certs are checked through
the read-only cert-status endpoint so the probe never triggers issuance. An
expired or unreadable cert counts as present, as failing readiness on it would
stop a StatefulSet rollout before it reaches the replica that renews certs.

Updates tailscale/corp#38175

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
chaosinthecrd authored and Tom Meadows committed 2026-11-19 13:17:44 +00:00
1 parent c4d689ce42
commit b5fa6773ce
9 files changed
+387 -26

No files matched your search

+130
View File
@@ -0,0 +1,130 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
"tailscale.com/kube/certs"
healthz "tailscale.com/kube/health"
"tailscale.com/tailcfg"
)
// Names of the checks served at /livez and /readyz.
const (
// checkTailscaled is a liveness check that tailscaled answers LocalAPI
// requests.
checkTailscaled = "tailscaled"
// checkNetmap is a readiness condition that this node has Tailscale IPs
// and the proxy rules for them are installed.
checkNetmap = "netmap"
// checkServe is a readiness condition that the serve config file has been
// applied.
checkServe = "serve"
// checkCerts is a readiness check that tailscaled's cert store holds a
// TLS cert for each HTTPS-only Tailscale Service this node advertises.
checkCerts = "certs"
)
// Reasons the checks give when failing.
const (
reasonNoTailscaleIPs = "node currently has no Tailscale IPs"
reasonServeNotApplied = "serve config not yet applied"
reasonCertsNotPresent = "TLS certs not yet present for all advertised Tailscale Services"
)
// healthLocalClient is the subset of [local.Client] used by the health checks.
type healthLocalClient interface {
StatusWithoutPeers(context.Context) (*ipnstate.Status, error)
GetPrefs(context.Context) (*ipn.Prefs, error)
GetServeConfig(context.Context) (*ipn.ServeConfig, error)
}
// readinessConditions are the /readyz conditions set by the code that owns
// each piece of state. A condition is nil if this proxy doesn't register it.
type readinessConditions struct {
netmap *healthz.Condition // set by the run loop on each netmap
serve *healthz.Condition // set by watchServeConfigChanges
}
// registerHealthChecks registers the /livez and /readyz probes and their
// checks on mux. It must be called before mux is served.
func registerHealthChecks(mux *http.ServeMux, cfg *settings, lc healthLocalClient, cm *certs.CertManager) readinessConditions {
log.Printf("Running liveness and readiness endpoints at %s/livez and %s/readyz", cfg.LocalAddrPort, cfg.LocalAddrPort)
lz := healthz.RegisterLivezHandlers(mux, log.Printf)
lz.AddCheck(checkTailscaled, tailscaledResponding(lc))
rz := healthz.RegisterReadyzHandlers(mux, log.Printf)
var rc readinessConditions
rc.netmap = rz.AddCondition(checkNetmap, reasonNoTailscaleIPs)
if cfg.ServeConfigPath != "" {
rc.serve = rz.AddCondition(checkServe, reasonServeNotApplied)
rz.AddCheck(checkCerts, certsPresent(cm, lc))
}
return rc
}
// tailscaledResponding returns a check that passes if tailscaled answers a
// LocalAPI status request.
func tailscaledResponding(lc healthLocalClient) func(context.Context) error {
return func(ctx context.Context) error {
if _, err := lc.StatusWithoutPeers(ctx); err != nil {
return fmt.Errorf("failed to get status: %w", err)
}
return nil
}
}
// certsPresent returns a check that passes if tailscaled's cert store holds a
// TLS cert for each HTTPS-only Tailscale Service in its serve config that this
// node advertises. The operator only advertises an HTTPS-only Tailscale Service
// once its cert has been issued, so one stuck on issuance doesn't fail every
// replica.
func certsPresent(cm *certs.CertManager, lc healthLocalClient) func(context.Context) error {
return func(ctx context.Context) error {
sc, err := lc.GetServeConfig(ctx)
if err != nil {
return fmt.Errorf("failed to get serve config: %w", err)
}
prefs, err := lc.GetPrefs(ctx)
if err != nil {
return fmt.Errorf("failed to get prefs: %w", err)
}
ok, err := cm.AllCertsPresent(ctx, advertisedServices(sc, prefs.AdvertiseServices))
switch {
case err != nil:
return fmt.Errorf("failed to check TLS certs: %w", err)
case !ok:
return errors.New(reasonCertsNotPresent)
}
return nil
}
}
// advertisedServices returns a copy of sc with only the Tailscale Services in
// advertised that don't also serve HTTP.
func advertisedServices(sc *ipn.ServeConfig, advertised []string) *ipn.ServeConfig {
out := *sc
out.Services = make(map[tailcfg.ServiceName]*ipn.ServiceConfig)
for _, svc := range advertised {
cfg, ok := sc.Services[tailcfg.ServiceName(svc)]
if !ok {
continue
}
// The operator advertises Services with an HTTP endpoint before their cert is issued.
if h := cfg.TCP[80]; h != nil && h.HTTP {
continue
}
out.Services[tailcfg.ServiceName(svc)] = cfg
}
return &out
}
+17 -5
View File
@@ -74,10 +74,8 @@
// - TS_ENABLE_METRICS: if true, a metrics endpoint will be served at /metrics on
// the address specified by TS_LOCAL_ADDR_PORT. See https://tailscale.com/kb/1482/client-metrics
// for more information on the metrics exposed.
// - TS_ENABLE_HEALTH_CHECK: if true, a health check endpoint will be served at /healthz on
// the address specified by TS_LOCAL_ADDR_PORT. The health endpoint will return 200
// OK if this node has at least one tailnet IP address, otherwise returns 503.
// NB: the health criteria might change in the future.
// - TS_ENABLE_HEALTH_CHECK: if true, the health check endpoints described below are
// served on the address specified by TS_LOCAL_ADDR_PORT.
// - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
// directory that containers tailscaled config in file. The config file needs to be
// named cap-<current-tailscaled-cap>.hujson. If this is set, TS_HOSTNAME,
@@ -109,6 +107,15 @@
// initial map response wait during boot. Accepts any time.Duration
// string (e.g. "90s", "3m").
//
// The health check endpoints return 200 OK while all of their checks pass,
// otherwise 503. NB: the health criteria might change in the future.
//
// - /healthz: this node has at least one tailnet IP address.
// - /livez: for Kubernetes liveness probes. tailscaled answers LocalAPI requests.
// - /readyz: for Kubernetes readiness probes. This node has tailnet IPs and, with
// TS_SERVE_CONFIG, the serve config is applied and there is a TLS cert for each
// advertised HTTPS Service.
//
// When running on Kubernetes, containerboot defaults to storing state in the
// "tailscale" kube secret. To store state on local disk instead, set
// TS_KUBE_SECRET="" and TS_STATE_DIR=/path/to/storage/dir. The state dir should
@@ -148,6 +155,7 @@ import (
"tailscale.com/ipn/ipnstate"
kubeutils "tailscale.com/k8s-operator"
"tailscale.com/kube/authkey"
"tailscale.com/kube/certs"
healthz "tailscale.com/kube/health"
"tailscale.com/kube/kubetypes"
klc "tailscale.com/kube/localclient"
@@ -511,6 +519,7 @@ func run() error {
defer killTailscaled()
var healthCheck *healthz.Healthz
var readiness readinessConditions
ep := &egressProxy{}
if cfg.HealthCheckAddrPort != "" {
mux := http.NewServeMux()
@@ -533,6 +542,8 @@ func run() error {
if cfg.localHealthEnabled() {
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, cfg.PodIPv6, log.Printf)
readiness = registerHealthChecks(mux, cfg, client, certs.NewCertManager(klc.New(client), log.Printf))
}
if cfg.egressSvcsTerminateEPEnabled() {
@@ -988,6 +999,7 @@ runLoop:
if healthCheck != nil {
healthCheck.Update(len(addrs) != 0)
}
readiness.netmap.Set(len(addrs) != 0)
var prevServeConfig *ipn.ServeConfig
if getAutoAdvertiseBool() {
@@ -1004,7 +1016,7 @@ runLoop:
if cfg.ServeConfigPath != "" {
triggerWatchServeConfigChanges.Do(func() {
go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig)
go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig, readiness.serve)
})
}
+7 -1
View File
@@ -21,6 +21,7 @@ import (
"tailscale.com/client/local"
"tailscale.com/ipn"
"tailscale.com/kube/certs"
healthz "tailscale.com/kube/health"
"tailscale.com/kube/kubetypes"
klc "tailscale.com/kube/localclient"
"tailscale.com/kube/services"
@@ -32,7 +33,7 @@ import (
// is written to when the certDomain changes, causing the serve config to be
// re-read and applied. prevServeConfig is the serve config that was fetched
// during startup. This will be refreshed by the goroutine when serve config changes.
func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings, prevServeConfig *ipn.ServeConfig) {
func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings, prevServeConfig *ipn.ServeConfig, applied *healthz.Condition) {
if certDomainAtomic == nil {
panic("certDomainAtomic must not be nil")
}
@@ -88,9 +89,13 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom
}
if sc == nil {
log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath)
// Nothing to serve, e.g. a ProxyGroup that only fronts Services.
applied.Set(true)
continue
}
if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
// Already applied, e.g. persisted in state from before a restart.
applied.Set(true)
continue
}
if err := updateServeConfig(ctx, sc, certDomain, klc.New(lc)); err != nil {
@@ -102,6 +107,7 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom
}
}
prevServeConfig = sc
applied.Set(true)
if cfg.CertShareMode != "rw" {
continue
}
+56 -20
View File
@@ -53,26 +53,9 @@ func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig)
if sc == nil {
return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
}
currentDomains := make(map[string]bool)
const httpsPort = "443"
for _, service := range sc.Services {
// L7 Web handlers (HA Ingress).
for hostPort := range service.Web {
domain, port, err := net.SplitHostPort(string(hostPort))
if err != nil {
return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
}
if port != httpsPort { // HA Ingress' HTTP endpoint
continue
}
currentDomains[domain] = true
}
// L4 TCP handlers with TLS termination (kube-apiserver proxy).
for _, handler := range service.TCP {
if handler != nil && handler.TerminateTLS != "" {
currentDomains[handler.TerminateTLS] = true
}
}
currentDomains, err := certDomainsForServeConfig(sc)
if err != nil {
return err
}
cm.mu.Lock()
defer cm.mu.Unlock()
@@ -97,6 +80,59 @@ func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig)
return nil
}
// certDomainsForServeConfig returns the set of domains that need a TLS cert for
// the HTTPS endpoints in sc.
func certDomainsForServeConfig(sc *ipn.ServeConfig) (map[string]bool, error) {
domains := make(map[string]bool)
const httpsPort = "443"
for _, service := range sc.Services {
// L7 Web handlers (HA Ingress).
for hostPort := range service.Web {
domain, port, err := net.SplitHostPort(string(hostPort))
if err != nil {
return nil, fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
}
if port != httpsPort { // HA Ingress' HTTP endpoint
continue
}
domains[domain] = true
}
// L4 TCP handlers with TLS termination (kube-apiserver proxy).
for _, handler := range service.TCP {
if handler != nil && handler.TerminateTLS != "" {
domains[handler.TerminateTLS] = true
}
}
}
return domains, nil
}
// AllCertsPresent reports whether a TLS cert is already cached for every HTTPS
// endpoint in sc. It reads cached certs only and never triggers issuance,
// so it is safe to call from a readiness probe. A ServeConfig with no HTTPS
// endpoints reports true. Only a missing cert reports false: an expired cert or
// a failed read from the cert store counts as present.
func (cm *CertManager) AllCertsPresent(ctx context.Context, sc *ipn.ServeConfig) (bool, error) {
if sc == nil {
return true, nil
}
domains, err := certDomainsForServeConfig(sc)
if err != nil {
return false, err
}
for domain := range domains {
info, err := cm.lc.CertStatus(ctx, domain)
if err != nil {
return false, fmt.Errorf("failed to get cert status for %q: %w", domain, err)
}
// Expired counts: else a rollout stalls before the replica that renews.
if info.Missing {
return false, nil
}
}
return true, nil
}
// isTransientCertErr reports whether err represents a failure that did not
// reach the CA (ctx timeout, LocalAPI socket unreachable). Such errors must
// not advance the loop's retryCount.
+24
View File
@@ -0,0 +1,24 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package health
import (
"net/http"
"tailscale.com/types/logger"
)
// RegisterLivezHandlers registers a liveness probe at /livez, which fails so
// that Kubernetes restarts the container. Only register checks for states a
// restart fixes: a proxy that has lost its connection to control, or is
// waiting for the operator to issue a new auth key, is still live. Register
// every check on the returned Probe before serving mux, as a Probe with no
// checks passes.
func RegisterLivezHandlers(mux *http.ServeMux, logf logger.Logf) *Probe {
p := &Probe{name: "livez", logf: logf}
mux.Handle("GET /livez", p)
return p
}
+112
View File
@@ -0,0 +1,112 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package health
import (
"context"
"errors"
"fmt"
"net/http"
"slices"
"sync"
"tailscale.com/types/logger"
)
// Probe is a Kubernetes probe server. It serves 200 only once every check
// registered with it passes, otherwise 503 with the reason each failing check
// gave.
//
// A nil Probe is valid and ignores registrations, so callers don't need to
// check whether health checks are enabled.
type Probe struct {
name string
logf logger.Logf
mu sync.Mutex // protects following
checks []check
}
type check struct {
name string
fn func(context.Context) error
}
func (p *Probe) ServeHTTP(w http.ResponseWriter, r *http.Request) {
p.mu.Lock()
checks := slices.Clone(p.checks)
p.mu.Unlock()
var errs []error
for _, c := range checks {
if err := c.fn(r.Context()); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", c.name, err))
}
}
if err := errors.Join(errs...); err != nil {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
if _, err := w.Write([]byte("ok")); err != nil {
http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError)
}
}
// AddCheck registers a check that is run on every request. fn returns nil if
// the check passes, or the reason it doesn't. fn must be cheap and safe for
// concurrent use.
func (p *Probe) AddCheck(name string, fn func(context.Context) error) {
if p == nil {
return
}
p.mu.Lock()
defer p.mu.Unlock()
p.checks = append(p.checks, check{name: name, fn: fn})
}
// AddCondition registers a check whose state is set by its owner with
// [Condition.Set]. It starts out failing, reporting reason.
func (p *Probe) AddCondition(name, reason string) *Condition {
if p == nil {
return nil
}
c := &Condition{name: p.name + ": " + name, reason: reason, logf: p.logf}
p.AddCheck(name, c.check)
return c
}
// Condition is a check that its owner sets as its state changes.
// A nil Condition is valid and ignores Set.
type Condition struct {
name string
reason string
logf logger.Logf
mu sync.Mutex // protects following
pass bool
}
// Set records whether the condition holds.
func (c *Condition) Set(pass bool) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if c.pass != pass {
c.logf("%s passing: %v", c.name, pass)
}
c.pass = pass
}
func (c *Condition) check(context.Context) error {
c.mu.Lock()
defer c.mu.Unlock()
if !c.pass {
return errors.New(c.reason)
}
return nil
}
+22
View File
@@ -0,0 +1,22 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package health
import (
"net/http"
"tailscale.com/types/logger"
)
// RegisterReadyzHandlers registers a readiness probe at /readyz, so each part
// of the proxy can keep the Pod out of its Service's endpoints until it can
// carry traffic. Register every check on the returned Probe before serving
// mux, as a Probe with no checks passes.
func RegisterReadyzHandlers(mux *http.ServeMux, logf logger.Logf) *Probe {
p := &Probe{name: "readyz", logf: logf}
mux.Handle("GET /readyz", p)
return p
}
+11
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"tailscale.com/ipn"
"tailscale.com/tailcfg"
)
type FakeLocalClient struct {
@@ -16,6 +17,9 @@ type FakeLocalClient struct {
EditPrefsCalls []*ipn.MaskedPrefs
GetPrefsResult *ipn.Prefs
CertDomainsResult []string
// CertStatusResults maps a domain to the cert status the fake returns.
// Domains absent from the map are reported as missing.
CertStatusResults map[string]*tailcfg.C2NTLSCertInfo
}
func (m *FakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error {
@@ -46,6 +50,13 @@ func (f *FakeLocalClient) CertPair(ctx context.Context, domain string) ([]byte,
return nil, nil, fmt.Errorf("CertPair not implemented")
}
func (f *FakeLocalClient) CertStatus(ctx context.Context, domain string) (*tailcfg.C2NTLSCertInfo, error) {
if info, ok := f.CertStatusResults[domain]; ok {
return info, nil
}
return &tailcfg.C2NTLSCertInfo{Missing: true, Error: "no certificate"}, nil
}
func (f *FakeLocalClient) CertDomains(ctx context.Context) ([]string, error) {
return f.CertDomainsResult, nil
}
+8
View File
@@ -11,6 +11,7 @@ import (
"tailscale.com/client/local"
"tailscale.com/ipn"
"tailscale.com/tailcfg"
)
// LocalClient is roughly a subset of the local.Client struct's methods, used
@@ -32,6 +33,9 @@ type IPNBusWatcher interface {
type CertIssuer interface {
CertPair(context.Context, string) ([]byte, []byte, error)
// CertStatus reports the state of the cached cert for domain without
// fetching or issuing one.
CertStatus(context.Context, string) (*tailcfg.C2NTLSCertInfo, error)
}
// New returns a LocalClient that wraps the provided local.Client.
@@ -59,6 +63,10 @@ func (lc *localClient) CertPair(ctx context.Context, domain string) ([]byte, []b
return lc.lc.CertPair(ctx, domain)
}
func (lc *localClient) CertStatus(ctx context.Context, domain string) (*tailcfg.C2NTLSCertInfo, error) {
return lc.lc.CertStatus(ctx, domain)
}
func (lc *localClient) CertDomains(ctx context.Context) ([]string, error) {
return lc.lc.CertDomains(ctx)
}