From 6308edfdce3d259123e8a4606e79e7d0f4713757 Mon Sep 17 00:00:00 2001 From: chaosinthecrd Date: Fri, 9 Oct 2026 16:08:56 +0000 Subject: [PATCH] cmd/k8s-operator,k8s-operator: run liveness and readiness probes on ProxyGroups Proxies now serve /livez and /readyz, but nothing points Kubernetes probes at them. This adds livenessProbe and readinessProbe fields to the ProxyClass Tailscale container, each off unless set, with tunable timing (older proxy images don't serve the endpoints, so a probe would fail against them), and wires them into ProxyGroup proxies. A ProxyGroup is rejected if its ProxyClass sets probes but its env would leave the endpoints unreachable, such as TS_ENABLE_HEALTH_CHECK=false, TS_LOCAL_ADDR_PORT on loopback, or the deprecated TS_HEALTHCHECK_ADDR_PORT. Probes follow a custom TS_LOCAL_ADDR_PORT port, and liveness gets a 5s timeout by default, as a restart drops traffic that a slow tailscaled still carries. Updates tailscale/corp#38175 Signed-off-by: chaosinthecrd --- .../crds/tailscale.com_proxyclasses.yaml | 148 ++++++++++++++++++ .../deploy/manifests/operator.yaml | 148 ++++++++++++++++++ cmd/k8s-operator/proxygroup.go | 17 ++ cmd/k8s-operator/proxygroup_specs.go | 5 +- cmd/k8s-operator/sts.go | 103 ++++++++++++ cmd/vet/jsontags_allowlist | 6 + k8s-operator/api.md | 23 +++ .../apis/v1alpha1/types_proxyclass.go | 41 +++++ .../apis/v1alpha1/zz_generated.deepcopy.go | 25 +++ .../reconciler/proxyclass/proxyclass.go | 8 + 10 files changed, 522 insertions(+), 2 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 73576b281..2febaa6fa 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -1456,6 +1456,80 @@ spec: - Always - Never - IfNotPresent + livenessProbe: + description: |- + Liveness probe against the proxy's /livez endpoint, which fails only when + tailscaled stops responding, not when the tailnet is unreachable. Off + unless set; {} runs it with the default timing. Older proxy images don't + serve /livez and fail the probe. Only used by ProxyGroups. + type: object + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + type: integer + format: int32 + default: 3 + minimum: 1 + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + type: integer + format: int32 + minimum: 0 + periodSeconds: + description: How often, in seconds, to perform the probe. + type: integer + format: int32 + default: 10 + minimum: 1 + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + type: integer + format: int32 + minimum: 1 + readinessProbe: + description: |- + Readiness probe against the proxy's /readyz endpoint, which passes once + the proxy has a tailnet IP address and, for proxies that serve Tailscale + Services, has applied its serve config and has a TLS cert for each + HTTPS-only Tailscale Service. Off unless set; {} runs it with the default + timing. Older proxy images don't serve /readyz and fail the probe. + Only used by ProxyGroups. + type: object + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + type: integer + format: int32 + default: 3 + minimum: 1 + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + type: integer + format: int32 + minimum: 0 + periodSeconds: + description: How often, in seconds, to perform the probe. + type: integer + format: int32 + default: 10 + minimum: 1 + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + type: integer + format: int32 + minimum: 1 resources: description: |- Container resource requirements. @@ -1796,6 +1870,80 @@ spec: - Always - Never - IfNotPresent + livenessProbe: + description: |- + Liveness probe against the proxy's /livez endpoint, which fails only when + tailscaled stops responding, not when the tailnet is unreachable. Off + unless set; {} runs it with the default timing. Older proxy images don't + serve /livez and fail the probe. Only used by ProxyGroups. + type: object + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + type: integer + format: int32 + default: 3 + minimum: 1 + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + type: integer + format: int32 + minimum: 0 + periodSeconds: + description: How often, in seconds, to perform the probe. + type: integer + format: int32 + default: 10 + minimum: 1 + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + type: integer + format: int32 + minimum: 1 + readinessProbe: + description: |- + Readiness probe against the proxy's /readyz endpoint, which passes once + the proxy has a tailnet IP address and, for proxies that serve Tailscale + Services, has applied its serve config and has a TLS cert for each + HTTPS-only Tailscale Service. Off unless set; {} runs it with the default + timing. Older proxy images don't serve /readyz and fail the probe. + Only used by ProxyGroups. + type: object + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + type: integer + format: int32 + default: 3 + minimum: 1 + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + type: integer + format: int32 + minimum: 0 + periodSeconds: + description: How often, in seconds, to perform the probe. + type: integer + format: int32 + default: 10 + minimum: 1 + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + type: integer + format: int32 + minimum: 1 resources: description: |- Container resource requirements. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 0fef30246..a71fb6ea7 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -3259,6 +3259,80 @@ spec: - Never - IfNotPresent type: string + livenessProbe: + description: |- + Liveness probe against the proxy's /livez endpoint, which fails only when + tailscaled stops responding, not when the tailnet is unreachable. Off + unless set; {} runs it with the default timing. Older proxy images don't + serve /livez and fail the probe. Only used by ProxyGroups. + properties: + failureThreshold: + default: 3 + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + format: int32 + minimum: 0 + type: integer + periodSeconds: + default: 10 + description: How often, in seconds, to perform the probe. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + format: int32 + minimum: 1 + type: integer + type: object + readinessProbe: + description: |- + Readiness probe against the proxy's /readyz endpoint, which passes once + the proxy has a tailnet IP address and, for proxies that serve Tailscale + Services, has applied its serve config and has a TLS cert for each + HTTPS-only Tailscale Service. Off unless set; {} runs it with the default + timing. Older proxy images don't serve /readyz and fail the probe. + Only used by ProxyGroups. + properties: + failureThreshold: + default: 3 + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + format: int32 + minimum: 0 + type: integer + periodSeconds: + default: 10 + description: How often, in seconds, to perform the probe. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + format: int32 + minimum: 1 + type: integer + type: object resources: description: |- Container resource requirements. @@ -3599,6 +3673,80 @@ spec: - Never - IfNotPresent type: string + livenessProbe: + description: |- + Liveness probe against the proxy's /livez endpoint, which fails only when + tailscaled stops responding, not when the tailnet is unreachable. Off + unless set; {} runs it with the default timing. Older proxy images don't + serve /livez and fail the probe. Only used by ProxyGroups. + properties: + failureThreshold: + default: 3 + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + format: int32 + minimum: 0 + type: integer + periodSeconds: + default: 10 + description: How often, in seconds, to perform the probe. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + format: int32 + minimum: 1 + type: integer + type: object + readinessProbe: + description: |- + Readiness probe against the proxy's /readyz endpoint, which passes once + the proxy has a tailnet IP address and, for proxies that serve Tailscale + Services, has applied its serve config and has a TLS cert for each + HTTPS-only Tailscale Service. Off unless set; {} runs it with the default + timing. Older proxy images don't serve /readyz and fail the probe. + Only used by ProxyGroups. + properties: + failureThreshold: + default: 3 + description: |- + Minimum consecutive failures for the probe to be considered failed after + having succeeded. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before the probe is + initiated. + format: int32 + minimum: 0 + type: integer + periodSeconds: + default: 10 + description: How often, in seconds, to perform the probe. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. Defaults to 5 for + livenessProbe and 1 for readinessProbe. + format: int32 + minimum: 1 + type: integer + type: object resources: description: |- Container resource requirements. diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index bea9fc21b..d0c7b9627 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -317,6 +317,18 @@ func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGrou if strings.HasSuffix(imagePath, "k8s-proxy") { errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type)) } + + if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleContainer != nil { + if tc := pc.Spec.StatefulSet.Pod.TailscaleContainer; tc.LivenessProbe != nil || tc.ReadinessProbe != nil { + var env []corev1.EnvVar + for _, e := range tc.Env { + env = append(env, corev1.EnvVar{Name: string(e.Name), Value: e.Value}) + } + if _, ok := healthzPort(env); !ok { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q sets probes, but its TS_HEALTHCHECK_ADDR_PORT, TS_ENABLE_HEALTH_CHECK or TS_LOCAL_ADDR_PORT env var turns off the /livez and /readyz endpoints or binds them where the kubelet can't reach them", pc.Name)) + } + } + } } return errors.Join(errs...) @@ -447,6 +459,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl } ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger) + // TODO(ChaosInTheCRD): temporary check, remove when probes are added for kube-apiserver ProxyGroups. + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + applyHealthzProbes(proxyClass, ss) + } + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec s.ObjectMeta.Labels = ss.ObjectMeta.Labels diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 79e9a2b61..04d68ff39 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -285,8 +285,9 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // accessible to the whole cluster network (in the absence of NetworkPolicy // rules). // - // TODO(tomhjp): add a readiness probe or gate to Ingress Pods. There is a - // small window where the Pod is marked ready but routing can still fail. + // TODO(tomhjp): run the readiness probe on Ingress Pods by default. Until a + // ProxyClass sets it, there is a small window where the Pod is marked ready + // but routing can still fail. if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && !hasLocalAddrPortSet(proxyClass) { c.Lifecycle = &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 33fd2c4b6..c660f5239 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -6,12 +6,14 @@ package main import ( + "cmp" "context" _ "embed" "encoding/json" "errors" "fmt" "maps" + "net" "net/netip" "os" "path" @@ -27,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apiserver/pkg/storage/names" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" @@ -1260,6 +1263,106 @@ func isMainContainer(c *corev1.Container) bool { return c.Name == mainContainerName } +// applyHealthzProbes attaches liveness and readiness probes against the proxy's +// /livez and /readyz endpoints to the main container, and enables those +// endpoints via TS_ENABLE_HEALTH_CHECK. Each probe is off unless the ProxyClass +// sets it. pc may be nil. It is a no-op if neither probe is set, or if the +// container's env leaves the endpoints unreachable, see [healthzPort]. +func applyHealthzProbes(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) { + var liveness, readiness *tsapi.Probe + if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleContainer != nil { + liveness = pc.Spec.StatefulSet.Pod.TailscaleContainer.LivenessProbe + readiness = pc.Spec.StatefulSet.Pod.TailscaleContainer.ReadinessProbe + } + if liveness == nil && readiness == nil { + return + } + + for i, c := range ss.Spec.Template.Spec.Containers { + if !isMainContainer(&c) { + continue + } + port, ok := healthzPort(c.Env) + if !ok { + return + } + if _, ok := lookupEnv(c.Env, "TS_ENABLE_HEALTH_CHECK"); !ok { + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, + corev1.EnvVar{Name: "TS_ENABLE_HEALTH_CHECK", Value: "true"}) + } + if liveness != nil { + // Longer than for readiness, as a restart drops the traffic a slow tailscaled still carries. + ss.Spec.Template.Spec.Containers[i].LivenessProbe = httpProbe("/livez", port, liveness, 5) + } + if readiness != nil { + ss.Spec.Template.Spec.Containers[i].ReadinessProbe = httpProbe("/readyz", port, readiness, 1) + } + break + } +} + +// healthzPort returns the port the main container serves /livez and /readyz +// on given its env. It returns false if the env turns the endpoints off, or +// binds them where the kubelet can't reach them, as probes would then restart +// a healthy proxy. +func healthzPort(env []corev1.EnvVar) (int, bool) { + if _, ok := lookupEnv(env, "TS_HEALTHCHECK_ADDR_PORT"); ok { + // containerboot refuses to start with both this and TS_ENABLE_HEALTH_CHECK. + return 0, false + } + if v, ok := lookupEnv(env, "TS_ENABLE_HEALTH_CHECK"); ok { + if on, err := strconv.ParseBool(v); err != nil || !on { + return 0, false + } + } + v, ok := lookupEnv(env, envVarTSLocalAddrPort) + if !ok { + return defaultLocalAddrPort, true + } + // The host may be an env var reference such as $(POD_IP). + host, port, err := net.SplitHostPort(v) + if err != nil { + return 0, false + } + if ip, err := netip.ParseAddr(host); host == "localhost" || err == nil && ip.IsLoopback() { + return 0, false + } + p, err := strconv.ParseUint(port, 10, 16) + if err != nil || p == 0 { + return 0, false + } + return int(p), true +} + +// lookupEnv returns the value of the last env var in env called name, as +// Kubernetes uses the last of any duplicates. +func lookupEnv(env []corev1.EnvVar, name string) (string, bool) { + for _, e := range slices.Backward(env) { + if e.Name == name { + return e.Value, true + } + } + return "", false +} + +// httpProbe builds an httpGet probe against path on port, using timeout and the +// Kubernetes defaults for unset timing fields. +func httpProbe(path string, port int, p *tsapi.Probe, timeout int32) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: path, + Port: intstr.FromInt(port), + }, + }, + InitialDelaySeconds: p.InitialDelaySeconds, + PeriodSeconds: cmp.Or(p.PeriodSeconds, 10), + TimeoutSeconds: cmp.Or(p.TimeoutSeconds, timeout), + FailureThreshold: cmp.Or(p.FailureThreshold, 3), + SuccessThreshold: 1, + } +} + // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy // state and auth key and returns tailscaled config files for currently supported proxy versions. func tailscaledConfig(stsC *tailscaleSTSConfig, loginUrl string, newAuthkey string, oldSecret *corev1.Secret, hostname string, staticEndpoints []netip.AddrPort) (tailscaledConfigs, error) { diff --git a/cmd/vet/jsontags_allowlist b/cmd/vet/jsontags_allowlist index 8d44c6e6c..2b990c034 100644 --- a/cmd/vet/jsontags_allowlist +++ b/cmd/vet/jsontags_allowlist @@ -41,6 +41,8 @@ OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.ConnectorSpec OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.ConnectorSpec.SubnetRouter OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.Container.Debug OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.Container.ImagePullPolicy +OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.Container.LivenessProbe +OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.Container.ReadinessProbe OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.Container.SecurityContext OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.KubeAPIServerConfig.Mode OmitEmptyShouldBeOmitZero tailscale.com/k8s-operator/apis/v1alpha1.Nameserver.Image @@ -215,6 +217,10 @@ OmitEmptyUnsupportedInV2 tailscale.com/ipn/ipnstate.PingResult.IsLocalIP OmitEmptyUnsupportedInV2 tailscale.com/ipn/ipnstate.PingResult.PeerAPIPort OmitEmptyUnsupportedInV2 tailscale.com/ipn/ipnstate.Status.HaveNodeKey OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.PortRange.EndPort +OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.Probe.FailureThreshold +OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.Probe.InitialDelaySeconds +OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.Probe.PeriodSeconds +OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.Probe.TimeoutSeconds OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.ProxyClassSpec.UseLetsEncryptStagingEnvironment OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.RecorderSpec.EnableUI OmitEmptyUnsupportedInV2 tailscale.com/k8s-operator/apis/v1alpha1.TailscaleConfig.AcceptRoutes diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 20822fa78..3c869029f 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -195,6 +195,8 @@ _Appears in:_ | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#resourcerequirements-v1-core)_ | Container resource requirements.
By default Tailscale Kubernetes operator does not apply any resource
requirements. The amount of resources required wil depend on the
amount of resources the operator needs to parse, usage patterns and
cluster size.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources | | Optional: \{\}
| | `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#securitycontext-v1-core)_ | Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context | | Optional: \{\}
| | `debug` _[Debug](#debug)_ | Configuration for enabling extra debug information in the container.
Not recommended for production use. | | Optional: \{\}
| +| `livenessProbe` _[Probe](#probe)_ | Liveness probe against the proxy's /livez endpoint, which fails only when
tailscaled stops responding, not when the tailnet is unreachable. Off
unless set; \{\} runs it with the default timing. Older proxy images don't
serve /livez and fail the probe. Only used by ProxyGroups. | | Optional: \{\}
| +| `readinessProbe` _[Probe](#probe)_ | Readiness probe against the proxy's /readyz endpoint, which passes once
the proxy has a tailnet IP address and, for proxies that serve Tailscale
Services, has applied its serve config and has a TLS cert for each
HTTPS-only Tailscale Service. Off unless set; \{\} runs it with the default
timing. Older proxy images don't serve /readyz and fail the probe.
Only used by ProxyGroups. | | Optional: \{\}
| #### DNSConfig @@ -752,6 +754,27 @@ _Appears in:_ +#### Probe + + + +Probe tunes a Kubernetes probe against the proxy. The operator sets the +probe's target; the fields here match the same fields on a Kubernetes probe +and each falls back to a default when unset. + + + +_Appears in:_ +- [Container](#container) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `initialDelaySeconds` _integer_ | Number of seconds after the container has started before the probe is
initiated. | | Minimum: 0
Optional: \{\}
| +| `periodSeconds` _integer_ | How often, in seconds, to perform the probe. | 10 | Minimum: 1
Optional: \{\}
| +| `timeoutSeconds` _integer_ | Number of seconds after which the probe times out. Defaults to 5 for
livenessProbe and 1 for readinessProbe. | | Minimum: 1
Optional: \{\}
| +| `failureThreshold` _integer_ | Minimum consecutive failures for the probe to be considered failed after
having succeeded. | 3 | Minimum: 1
Optional: \{\}
| + + #### ProxyClass diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 7861fed94..4d2889ec6 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -426,6 +426,47 @@ type Container struct { // Not recommended for production use. // +optional Debug *Debug `json:"debug,omitempty"` + // Liveness probe against the proxy's /livez endpoint, which fails only when + // tailscaled stops responding, not when the tailnet is unreachable. Off + // unless set; {} runs it with the default timing. Older proxy images don't + // serve /livez and fail the probe. Only used by ProxyGroups. + // +optional + LivenessProbe *Probe `json:"livenessProbe,omitempty"` + // Readiness probe against the proxy's /readyz endpoint, which passes once + // the proxy has a tailnet IP address and, for proxies that serve Tailscale + // Services, has applied its serve config and has a TLS cert for each + // HTTPS-only Tailscale Service. Off unless set; {} runs it with the default + // timing. Older proxy images don't serve /readyz and fail the probe. + // Only used by ProxyGroups. + // +optional + ReadinessProbe *Probe `json:"readinessProbe,omitempty"` +} + +// Probe tunes a Kubernetes probe against the proxy. The operator sets the +// probe's target; the fields here match the same fields on a Kubernetes probe +// and each falls back to a default when unset. +type Probe struct { + // Number of seconds after the container has started before the probe is + // initiated. + // +optional + // +kubebuilder:validation:Minimum=0 + InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` + // How often, in seconds, to perform the probe. + // +optional + // +kubebuilder:default=10 + // +kubebuilder:validation:Minimum=1 + PeriodSeconds int32 `json:"periodSeconds,omitempty"` + // Number of seconds after which the probe times out. Defaults to 5 for + // livenessProbe and 1 for readinessProbe. + // +optional + // +kubebuilder:validation:Minimum=1 + TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` + // Minimum consecutive failures for the probe to be considered failed after + // having succeeded. + // +optional + // +kubebuilder:default=3 + // +kubebuilder:validation:Minimum=1 + FailureThreshold int32 `json:"failureThreshold,omitempty"` } type Debug struct { diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 91a09c73e..9a37b0294 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -205,6 +205,16 @@ func (in *Container) DeepCopyInto(out *Container) { *out = new(Debug) **out = **in } + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(Probe) + **out = **in + } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(Probe) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container. @@ -892,6 +902,21 @@ func (in PortRanges) DeepCopy() PortRanges { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Probe) DeepCopyInto(out *Probe) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Probe. +func (in *Probe) DeepCopy() *Probe { + if in == nil { + return nil + } + out := new(Probe) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyClass) DeepCopyInto(out *ProxyClass) { *out = *in diff --git a/k8s-operator/reconciler/proxyclass/proxyclass.go b/k8s-operator/reconciler/proxyclass/proxyclass.go index 80bd32b33..125afa99c 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass.go @@ -231,6 +231,14 @@ func (r *Reconciler) validate(ctx context.Context, pc *tsapi.ProxyClass, logger if tc.Debug != nil { violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "debug"), tc.Debug, "debug settings cannot be configured on the init container")) } + + if tc.LivenessProbe != nil { + violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "livenessProbe"), tc.LivenessProbe, "probes cannot be configured on the init container")) + } + + if tc.ReadinessProbe != nil { + violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "readinessProbe"), tc.ReadinessProbe, "probes cannot be configured on the init container")) + } } } }