diff --git a/cmd/containerboot/health.go b/cmd/containerboot/health.go new file mode 100644 index 000000000..51369d6ea --- /dev/null +++ b/cmd/containerboot/health.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/kube/certs" + healthz "tailscale.com/kube/health" + "tailscale.com/tailcfg" +) + +// Names of the checks served at /livez and /readyz. +const ( + // checkTailscaled is a liveness check that tailscaled answers LocalAPI + // requests. + checkTailscaled = "tailscaled" + // checkNetmap is a readiness condition that this node has Tailscale IPs + // and the proxy rules for them are installed. + checkNetmap = "netmap" + // checkServe is a readiness condition that the serve config file has been + // applied. + checkServe = "serve" + // checkCerts is a readiness check that tailscaled's cert store holds a + // TLS cert for each HTTPS-only Tailscale Service this node advertises. + checkCerts = "certs" +) + +// Reasons the checks give when failing. +const ( + reasonNoTailscaleIPs = "node currently has no Tailscale IPs" + reasonServeNotApplied = "serve config not yet applied" + reasonCertsNotPresent = "TLS certs not yet present for all advertised Tailscale Services" +) + +// healthLocalClient is the subset of [local.Client] used by the health checks. +type healthLocalClient interface { + StatusWithoutPeers(context.Context) (*ipnstate.Status, error) + GetPrefs(context.Context) (*ipn.Prefs, error) + GetServeConfig(context.Context) (*ipn.ServeConfig, error) +} + +// readinessConditions are the /readyz conditions set by the code that owns +// each piece of state. A condition is nil if this proxy doesn't register it. +type readinessConditions struct { + netmap *healthz.Condition // set by the run loop on each netmap + serve *healthz.Condition // set by watchServeConfigChanges +} + +// registerHealthChecks registers the /livez and /readyz probes and their +// checks on mux. It must be called before mux is served. +func registerHealthChecks(mux *http.ServeMux, cfg *settings, lc healthLocalClient, cm *certs.CertManager) readinessConditions { + log.Printf("Running liveness and readiness endpoints at %s/livez and %s/readyz", cfg.LocalAddrPort, cfg.LocalAddrPort) + lz := healthz.RegisterLivezHandlers(mux, log.Printf) + lz.AddCheck(checkTailscaled, tailscaledResponding(lc)) + + rz := healthz.RegisterReadyzHandlers(mux, log.Printf) + var rc readinessConditions + rc.netmap = rz.AddCondition(checkNetmap, reasonNoTailscaleIPs) + if cfg.ServeConfigPath != "" { + rc.serve = rz.AddCondition(checkServe, reasonServeNotApplied) + rz.AddCheck(checkCerts, certsPresent(cm, lc)) + } + return rc +} + +// tailscaledResponding returns a check that passes if tailscaled answers a +// LocalAPI status request. +func tailscaledResponding(lc healthLocalClient) func(context.Context) error { + return func(ctx context.Context) error { + if _, err := lc.StatusWithoutPeers(ctx); err != nil { + return fmt.Errorf("failed to get status: %w", err) + } + return nil + } +} + +// certsPresent returns a check that passes if tailscaled's cert store holds a +// TLS cert for each HTTPS-only Tailscale Service in its serve config that this +// node advertises. The operator only advertises an HTTPS-only Tailscale Service +// once its cert has been issued, so one stuck on issuance doesn't fail every +// replica. +func certsPresent(cm *certs.CertManager, lc healthLocalClient) func(context.Context) error { + return func(ctx context.Context) error { + sc, err := lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("failed to get serve config: %w", err) + } + prefs, err := lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("failed to get prefs: %w", err) + } + ok, err := cm.AllCertsPresent(ctx, advertisedServices(sc, prefs.AdvertiseServices)) + switch { + case err != nil: + return fmt.Errorf("failed to check TLS certs: %w", err) + case !ok: + return errors.New(reasonCertsNotPresent) + } + return nil + } +} + +// advertisedServices returns a copy of sc with only the Tailscale Services in +// advertised that don't also serve HTTP. +func advertisedServices(sc *ipn.ServeConfig, advertised []string) *ipn.ServeConfig { + out := *sc + out.Services = make(map[tailcfg.ServiceName]*ipn.ServiceConfig) + for _, svc := range advertised { + cfg, ok := sc.Services[tailcfg.ServiceName(svc)] + if !ok { + continue + } + // The operator advertises Services with an HTTP endpoint before their cert is issued. + if h := cfg.TCP[80]; h != nil && h.HTTP { + continue + } + out.Services[tailcfg.ServiceName(svc)] = cfg + } + return &out +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 84eb9d2c5..043bf5584 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -74,10 +74,8 @@ // - TS_ENABLE_METRICS: if true, a metrics endpoint will be served at /metrics on // the address specified by TS_LOCAL_ADDR_PORT. See https://tailscale.com/kb/1482/client-metrics // for more information on the metrics exposed. -// - TS_ENABLE_HEALTH_CHECK: if true, a health check endpoint will be served at /healthz on -// the address specified by TS_LOCAL_ADDR_PORT. The health endpoint will return 200 -// OK if this node has at least one tailnet IP address, otherwise returns 503. -// NB: the health criteria might change in the future. +// - TS_ENABLE_HEALTH_CHECK: if true, the health check endpoints described below are +// served on the address specified by TS_LOCAL_ADDR_PORT. // - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a // directory that containers tailscaled config in file. The config file needs to be // named cap-.hujson. If this is set, TS_HOSTNAME, @@ -109,6 +107,15 @@ // initial map response wait during boot. Accepts any time.Duration // string (e.g. "90s", "3m"). // +// The health check endpoints return 200 OK while all of their checks pass, +// otherwise 503. NB: the health criteria might change in the future. +// +// - /healthz: this node has at least one tailnet IP address. +// - /livez: for Kubernetes liveness probes. tailscaled answers LocalAPI requests. +// - /readyz: for Kubernetes readiness probes. This node has tailnet IPs and, with +// TS_SERVE_CONFIG, the serve config is applied and there is a TLS cert for each +// advertised HTTPS Service. +// // When running on Kubernetes, containerboot defaults to storing state in the // "tailscale" kube secret. To store state on local disk instead, set // TS_KUBE_SECRET="" and TS_STATE_DIR=/path/to/storage/dir. The state dir should @@ -148,6 +155,7 @@ import ( "tailscale.com/ipn/ipnstate" kubeutils "tailscale.com/k8s-operator" "tailscale.com/kube/authkey" + "tailscale.com/kube/certs" healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" @@ -511,6 +519,7 @@ func run() error { defer killTailscaled() var healthCheck *healthz.Healthz + var readiness readinessConditions ep := &egressProxy{} if cfg.HealthCheckAddrPort != "" { mux := http.NewServeMux() @@ -533,6 +542,8 @@ func run() error { if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, cfg.PodIPv6, log.Printf) + + readiness = registerHealthChecks(mux, cfg, client, certs.NewCertManager(klc.New(client), log.Printf)) } if cfg.egressSvcsTerminateEPEnabled() { @@ -988,6 +999,7 @@ runLoop: if healthCheck != nil { healthCheck.Update(len(addrs) != 0) } + readiness.netmap.Set(len(addrs) != 0) var prevServeConfig *ipn.ServeConfig if getAutoAdvertiseBool() { @@ -1004,7 +1016,7 @@ runLoop: if cfg.ServeConfigPath != "" { triggerWatchServeConfigChanges.Do(func() { - go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig) + go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg, prevServeConfig, readiness.serve) }) } diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index f5423630f..34d270dd7 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -21,6 +21,7 @@ import ( "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/kube/certs" + healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" "tailscale.com/kube/services" @@ -32,7 +33,7 @@ import ( // is written to when the certDomain changes, causing the serve config to be // re-read and applied. prevServeConfig is the serve config that was fetched // during startup. This will be refreshed by the goroutine when serve config changes. -func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings, prevServeConfig *ipn.ServeConfig) { +func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings, prevServeConfig *ipn.ServeConfig, applied *healthz.Condition) { if certDomainAtomic == nil { panic("certDomainAtomic must not be nil") } @@ -88,9 +89,13 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom } if sc == nil { log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath) + // Nothing to serve, e.g. a ProxyGroup that only fronts Services. + applied.Set(true) continue } if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) { + // Already applied, e.g. persisted in state from before a restart. + applied.Set(true) continue } if err := updateServeConfig(ctx, sc, certDomain, klc.New(lc)); err != nil { @@ -102,6 +107,7 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom } } prevServeConfig = sc + applied.Set(true) if cfg.CertShareMode != "rw" { continue } diff --git a/kube/certs/certs.go b/kube/certs/certs.go index ec8956b33..e1beceaf1 100644 --- a/kube/certs/certs.go +++ b/kube/certs/certs.go @@ -53,26 +53,9 @@ func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) if sc == nil { return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig") } - currentDomains := make(map[string]bool) - const httpsPort = "443" - for _, service := range sc.Services { - // L7 Web handlers (HA Ingress). - for hostPort := range service.Web { - domain, port, err := net.SplitHostPort(string(hostPort)) - if err != nil { - return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort) - } - if port != httpsPort { // HA Ingress' HTTP endpoint - continue - } - currentDomains[domain] = true - } - // L4 TCP handlers with TLS termination (kube-apiserver proxy). - for _, handler := range service.TCP { - if handler != nil && handler.TerminateTLS != "" { - currentDomains[handler.TerminateTLS] = true - } - } + currentDomains, err := certDomainsForServeConfig(sc) + if err != nil { + return err } cm.mu.Lock() defer cm.mu.Unlock() @@ -97,6 +80,59 @@ func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) return nil } +// certDomainsForServeConfig returns the set of domains that need a TLS cert for +// the HTTPS endpoints in sc. +func certDomainsForServeConfig(sc *ipn.ServeConfig) (map[string]bool, error) { + domains := make(map[string]bool) + const httpsPort = "443" + for _, service := range sc.Services { + // L7 Web handlers (HA Ingress). + for hostPort := range service.Web { + domain, port, err := net.SplitHostPort(string(hostPort)) + if err != nil { + return nil, fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort) + } + if port != httpsPort { // HA Ingress' HTTP endpoint + continue + } + domains[domain] = true + } + // L4 TCP handlers with TLS termination (kube-apiserver proxy). + for _, handler := range service.TCP { + if handler != nil && handler.TerminateTLS != "" { + domains[handler.TerminateTLS] = true + } + } + } + return domains, nil +} + +// AllCertsPresent reports whether a TLS cert is already cached for every HTTPS +// endpoint in sc. It reads cached certs only and never triggers issuance, +// so it is safe to call from a readiness probe. A ServeConfig with no HTTPS +// endpoints reports true. Only a missing cert reports false: an expired cert or +// a failed read from the cert store counts as present. +func (cm *CertManager) AllCertsPresent(ctx context.Context, sc *ipn.ServeConfig) (bool, error) { + if sc == nil { + return true, nil + } + domains, err := certDomainsForServeConfig(sc) + if err != nil { + return false, err + } + for domain := range domains { + info, err := cm.lc.CertStatus(ctx, domain) + if err != nil { + return false, fmt.Errorf("failed to get cert status for %q: %w", domain, err) + } + // Expired counts: else a rollout stalls before the replica that renews. + if info.Missing { + return false, nil + } + } + return true, nil +} + // isTransientCertErr reports whether err represents a failure that did not // reach the CA (ctx timeout, LocalAPI socket unreachable). Such errors must // not advance the loop's retryCount. diff --git a/kube/health/livez.go b/kube/health/livez.go new file mode 100644 index 000000000..59e5ffd5c --- /dev/null +++ b/kube/health/livez.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package health + +import ( + "net/http" + + "tailscale.com/types/logger" +) + +// RegisterLivezHandlers registers a liveness probe at /livez, which fails so +// that Kubernetes restarts the container. Only register checks for states a +// restart fixes: a proxy that has lost its connection to control, or is +// waiting for the operator to issue a new auth key, is still live. Register +// every check on the returned Probe before serving mux, as a Probe with no +// checks passes. +func RegisterLivezHandlers(mux *http.ServeMux, logf logger.Logf) *Probe { + p := &Probe{name: "livez", logf: logf} + mux.Handle("GET /livez", p) + return p +} diff --git a/kube/health/probe.go b/kube/health/probe.go new file mode 100644 index 000000000..b4d75a40c --- /dev/null +++ b/kube/health/probe.go @@ -0,0 +1,112 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package health + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + "sync" + + "tailscale.com/types/logger" +) + +// Probe is a Kubernetes probe server. It serves 200 only once every check +// registered with it passes, otherwise 503 with the reason each failing check +// gave. +// +// A nil Probe is valid and ignores registrations, so callers don't need to +// check whether health checks are enabled. +type Probe struct { + name string + logf logger.Logf + + mu sync.Mutex // protects following + checks []check +} + +type check struct { + name string + fn func(context.Context) error +} + +func (p *Probe) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p.mu.Lock() + checks := slices.Clone(p.checks) + p.mu.Unlock() + + var errs []error + for _, c := range checks { + if err := c.fn(r.Context()); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", c.name, err)) + } + } + if err := errors.Join(errs...); err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + if _, err := w.Write([]byte("ok")); err != nil { + http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) + } +} + +// AddCheck registers a check that is run on every request. fn returns nil if +// the check passes, or the reason it doesn't. fn must be cheap and safe for +// concurrent use. +func (p *Probe) AddCheck(name string, fn func(context.Context) error) { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.checks = append(p.checks, check{name: name, fn: fn}) +} + +// AddCondition registers a check whose state is set by its owner with +// [Condition.Set]. It starts out failing, reporting reason. +func (p *Probe) AddCondition(name, reason string) *Condition { + if p == nil { + return nil + } + c := &Condition{name: p.name + ": " + name, reason: reason, logf: p.logf} + p.AddCheck(name, c.check) + return c +} + +// Condition is a check that its owner sets as its state changes. +// A nil Condition is valid and ignores Set. +type Condition struct { + name string + reason string + logf logger.Logf + + mu sync.Mutex // protects following + pass bool +} + +// Set records whether the condition holds. +func (c *Condition) Set(pass bool) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.pass != pass { + c.logf("%s passing: %v", c.name, pass) + } + c.pass = pass +} + +func (c *Condition) check(context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + if !c.pass { + return errors.New(c.reason) + } + return nil +} diff --git a/kube/health/readyz.go b/kube/health/readyz.go new file mode 100644 index 000000000..b5c95f672 --- /dev/null +++ b/kube/health/readyz.go @@ -0,0 +1,22 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package health + +import ( + "net/http" + + "tailscale.com/types/logger" +) + +// RegisterReadyzHandlers registers a readiness probe at /readyz, so each part +// of the proxy can keep the Pod out of its Service's endpoints until it can +// carry traffic. Register every check on the returned Probe before serving +// mux, as a Probe with no checks passes. +func RegisterReadyzHandlers(mux *http.ServeMux, logf logger.Logf) *Probe { + p := &Probe{name: "readyz", logf: logf} + mux.Handle("GET /readyz", p) + return p +} diff --git a/kube/localclient/fake-client.go b/kube/localclient/fake-client.go index 7ecada113..542037aa2 100644 --- a/kube/localclient/fake-client.go +++ b/kube/localclient/fake-client.go @@ -8,6 +8,7 @@ import ( "fmt" "tailscale.com/ipn" + "tailscale.com/tailcfg" ) type FakeLocalClient struct { @@ -16,6 +17,9 @@ type FakeLocalClient struct { EditPrefsCalls []*ipn.MaskedPrefs GetPrefsResult *ipn.Prefs CertDomainsResult []string + // CertStatusResults maps a domain to the cert status the fake returns. + // Domains absent from the map are reported as missing. + CertStatusResults map[string]*tailcfg.C2NTLSCertInfo } func (m *FakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error { @@ -46,6 +50,13 @@ func (f *FakeLocalClient) CertPair(ctx context.Context, domain string) ([]byte, return nil, nil, fmt.Errorf("CertPair not implemented") } +func (f *FakeLocalClient) CertStatus(ctx context.Context, domain string) (*tailcfg.C2NTLSCertInfo, error) { + if info, ok := f.CertStatusResults[domain]; ok { + return info, nil + } + return &tailcfg.C2NTLSCertInfo{Missing: true, Error: "no certificate"}, nil +} + func (f *FakeLocalClient) CertDomains(ctx context.Context) ([]string, error) { return f.CertDomainsResult, nil } diff --git a/kube/localclient/local-client.go b/kube/localclient/local-client.go index f759568ba..541a98056 100644 --- a/kube/localclient/local-client.go +++ b/kube/localclient/local-client.go @@ -11,6 +11,7 @@ import ( "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/tailcfg" ) // LocalClient is roughly a subset of the local.Client struct's methods, used @@ -32,6 +33,9 @@ type IPNBusWatcher interface { type CertIssuer interface { CertPair(context.Context, string) ([]byte, []byte, error) + // CertStatus reports the state of the cached cert for domain without + // fetching or issuing one. + CertStatus(context.Context, string) (*tailcfg.C2NTLSCertInfo, error) } // New returns a LocalClient that wraps the provided local.Client. @@ -59,6 +63,10 @@ func (lc *localClient) CertPair(ctx context.Context, domain string) ([]byte, []b return lc.lc.CertPair(ctx, domain) } +func (lc *localClient) CertStatus(ctx context.Context, domain string) (*tailcfg.C2NTLSCertInfo, error) { + return lc.lc.CertStatus(ctx, domain) +} + func (lc *localClient) CertDomains(ctx context.Context) ([]string, error) { return lc.lc.CertDomains(ctx) }