From 9eea2c890c82ec90eca813be15cb3f5725bebc96 Mon Sep 17 00:00:00 2001 From: chaosinthecrd Date: Wed, 4 Nov 2026 20:37:16 +0000 Subject: [PATCH] cmd/k8s-operator,k8s-operator: support ProxyClass metrics for PeerRelay The PeerRelay reconciler ignored a ProxyClass's spec.metrics, so enabling metrics created no metrics Service or ServiceMonitor and the relay Pods never served /metrics. This fixes this by setting TS_ENABLE_METRICS in ApplyProxyClass and calling tailscaled.EnsureMetricsResources for PeerRelay. PeerRelay now also waits for its ProxyClass to be ready before creating anything, as the other proxies do, so the existing ProxyClass test fixture is marked ready. ApplyProxyClass now replaces an env var that the ProxyClass also sets instead of appending a second one. The PeerRelay StatefulSet is applied with server-side apply, which rejects duplicate env names, so a ProxyClass that sets an env var the operator also sets, such as TS_ENABLE_METRICS, would stop the StatefulSet from being applied. Updates #22034 Signed-off-by: chaosinthecrd --- .../crds/tailscale.com_proxyclasses.yaml | 4 +- .../deploy/manifests/operator.yaml | 4 +- k8s-operator/api.md | 2 +- .../apis/v1alpha1/types_proxyclass.go | 4 +- .../reconciler/peerrelay/peerrelay.go | 38 ++++++++++++++----- .../reconciler/peerrelay/peerrelay_test.go | 11 ++++++ .../reconciler/peerrelay/statefulset.go | 13 +++++++ k8s-operator/reconciler/tailscaled/metrics.go | 3 +- .../reconciler/tailscaled/proxyclass.go | 33 ++++++++++++++-- 9 files changed, 91 insertions(+), 21 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 6c56ed887..73576b281 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -91,8 +91,8 @@ spec: Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics. The ServiceMonitor will select the metrics Service that gets created when metrics are enabled. The ingested metrics for each Service monitor will have labels to identify the proxy: - ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup - ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup) + ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup|peerrelay + ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service, ProxyGroup or PeerRelay) ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped) job: ts__[]_ type: object diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index dd3d9c451..0fef30246 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -1894,8 +1894,8 @@ spec: Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics. The ServiceMonitor will select the metrics Service that gets created when metrics are enabled. The ingested metrics for each Service monitor will have labels to identify the proxy: - ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup - ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup) + ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup|peerrelay + ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service, ProxyGroup or PeerRelay) ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped) job: ts__[]_ properties: diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 3fdb68aad..20822fa78 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -429,7 +429,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `enable` _boolean_ | Setting enable to true will make the proxy serve Tailscale metrics
at :9002/metrics.
A metrics Service named -metrics will also be created in the operator's namespace and will
serve the metrics at :9002/metrics.
In 1.78.x and 1.80.x, this field also serves as the default value for
.spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
fields will independently default to false.
Defaults to false. | | | -| `serviceMonitor` _[ServiceMonitor](#servicemonitor)_ | Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
The ingested metrics for each Service monitor will have labels to identify the proxy:
ts_proxy_type: ingress_service\|ingress_resource\|connector\|proxygroup
ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
job: ts__[]_ | | Optional: \{\}
| +| `serviceMonitor` _[ServiceMonitor](#servicemonitor)_ | Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
The ingested metrics for each Service monitor will have labels to identify the proxy:
ts_proxy_type: ingress_service\|ingress_resource\|connector\|proxygroup\|peerrelay
ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service, ProxyGroup or PeerRelay)
ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
job: ts__[]_ | | Optional: \{\}
| #### Name diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 7e326e678..7861fed94 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -332,8 +332,8 @@ type Metrics struct { // Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics. // The ServiceMonitor will select the metrics Service that gets created when metrics are enabled. // The ingested metrics for each Service monitor will have labels to identify the proxy: - // ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup - // ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup) + // ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup|peerrelay + // ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service, ProxyGroup or PeerRelay) // ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped) // job: ts__[]_ // +optional diff --git a/k8s-operator/reconciler/peerrelay/peerrelay.go b/k8s-operator/reconciler/peerrelay/peerrelay.go index 134f087f8..02b557408 100644 --- a/k8s-operator/reconciler/peerrelay/peerrelay.go +++ b/k8s-operator/reconciler/peerrelay/peerrelay.go @@ -96,6 +96,7 @@ const ( ReasonAWSConfigInvalid = "AWSConfigInvalid" ReasonStaticEndpointsInvalid = "StaticEndpointsInvalid" ReasonTailnetUnavailable = "TailnetUnavailable" + ReasonProxyClassNotReady = "ProxyClassNotReady" ReasonReady = "PeerRelayReady" ) @@ -248,6 +249,22 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, logger *zap.SugaredLogg return reconcile.Result{}, nil } + pc, err := r.getProxyClass(ctx, pr) + if err != nil { + return reconcile.Result{}, err + } + + // The ProxyClass watch re-enqueues this PeerRelay once the ProxyClass becomes ready. + if pc != nil && !operatorutils.ProxyClassIsReady(pc) { + message := fmt.Sprintf("ProxyClass %q is not yet in a ready state, waiting...", pc.Name) + logger.Info(message) + operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonProxyClassNotReady, message, r.clock, logger) + if err := r.Status().Update(ctx, pr); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update PeerRelay status for %q: %w", pr.Name, err) + } + return reconcile.Result{}, nil + } + for i := int32(0); i < replicas; i++ { desired := r.peerRelayService(pr, i) if err := r.ensureService(ctx, logger, desired); err != nil { @@ -279,11 +296,15 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, logger *zap.SugaredLogg } } - ss, err := r.ensureStatefulSet(ctx, logger, pr, replicas) + ss, err := r.ensureStatefulSet(ctx, logger, pr, replicas, pc) if err != nil { return reconcile.Result{}, fmt.Errorf("failed to apply StatefulSet for PeerRelay %q: %w", pr.Name, err) } + if err = tailscaled.EnsureMetricsResources(ctx, r.Client, logger, r.metricsOptions(pr), pc); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to apply metrics resources for PeerRelay %q: %w", pr.Name, err) + } + if err = r.deleteDevicesFrom(ctx, logger, pr, replicas); err != nil { return reconcile.Result{}, fmt.Errorf("failed to clean up scaled-down tailnet devices for PeerRelay %q: %w", pr.Name, err) } @@ -493,6 +514,10 @@ func (r *Reconciler) delete(ctx context.Context, logger *zap.SugaredLogger, pr * return reconcile.Result{}, fmt.Errorf("failed to delete Services for PeerRelay %q: %w", pr.Name, err) } + if err := tailscaled.EnsureMetricsResourcesDeleted(ctx, r.Client, r.metricsOptions(pr)); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to delete metrics resources for PeerRelay %q: %w", pr.Name, err) + } + if err := reconciler.ClearFinalizer(ctx, r.Client, pr, reconciler.Finalizer); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from PeerRelay %q: %w", pr.Name, err) } @@ -597,21 +622,16 @@ func (r *Reconciler) deleteConfigSecretsFrom(ctx context.Context, logger *zap.Su return nil } -func (r *Reconciler) ensureStatefulSet(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, replicas int32) (*appsv1.StatefulSet, error) { - pc, err := r.getProxyClass(ctx, pr) - if err != nil { - return nil, err - } - +func (r *Reconciler) ensureStatefulSet(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, replicas int32, pc *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { desired := r.peerRelayStatefulSet(pr, replicas, pc) logger.Debugf("applying StatefulSet %q", desired.Name) - if err = r.Patch(ctx, desired, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + if err := r.Patch(ctx, desired, client.Apply, fieldOwner, client.ForceOwnership); err != nil { return nil, fmt.Errorf("failed to apply StatefulSet: %w", err) } var current appsv1.StatefulSet - if err = r.Get(ctx, types.NamespacedName{Namespace: desired.Namespace, Name: desired.Name}, ¤t); err != nil { + if err := r.Get(ctx, types.NamespacedName{Namespace: desired.Namespace, Name: desired.Name}, ¤t); err != nil { return nil, fmt.Errorf("failed to get StatefulSet: %w", err) } diff --git a/k8s-operator/reconciler/peerrelay/peerrelay_test.go b/k8s-operator/reconciler/peerrelay/peerrelay_test.go index 6254542da..0dbbeeedd 100644 --- a/k8s-operator/reconciler/peerrelay/peerrelay_test.go +++ b/k8s-operator/reconciler/peerrelay/peerrelay_test.go @@ -1874,6 +1874,7 @@ func TestReconciler_AppliesProxyClass(t *testing.T) { }, }, }, + Status: readyProxyClassStatus(), } fc := reconcilertest.NewClientBuilder().WithInterceptorFuncs(reconcilertest.ApplyPatchInterceptor()). @@ -1946,3 +1947,13 @@ func TestReconciler_AppliesProxyClass(t *testing.T) { t.Errorf("expected container to keep the NET_ADMIN capability, got security context %+v", c.SecurityContext) } } + +// readyProxyClassStatus returns a ready status; the fake client leaves Generation at 0, so ObservedGeneration is too. +func readyProxyClassStatus() tsapi.ProxyClassStatus { + return tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + }}, + } +} diff --git a/k8s-operator/reconciler/peerrelay/statefulset.go b/k8s-operator/reconciler/peerrelay/statefulset.go index 13303e971..cbcd06dbd 100644 --- a/k8s-operator/reconciler/peerrelay/statefulset.go +++ b/k8s-operator/reconciler/peerrelay/statefulset.go @@ -101,6 +101,19 @@ var managedLabelKeys = []string{ reconciler.LabelParentName, } +func (r *Reconciler) metricsOptions(pr *tsapi.PeerRelay) tailscaled.MetricsOptions { + return tailscaled.MetricsOptions{ + StatefulSetName: resourceName(pr.Name), + Namespace: r.tailscaleNamespace, + PodLabels: peerRelayLabels(pr.Name), + ProxyType: parentTypePeerRelay, + ParentName: pr.Name, + + // Lets the Service watch enqueue this PeerRelay when its metrics Service changes. + Labels: peerRelayLabels(pr.Name), + } +} + func (r *Reconciler) getProxyClass(ctx context.Context, pr *tsapi.PeerRelay) (*tsapi.ProxyClass, error) { if pr.Spec.ProxyClass == "" { return nil, nil diff --git a/k8s-operator/reconciler/tailscaled/metrics.go b/k8s-operator/reconciler/tailscaled/metrics.go index 40faa62b1..7530ca480 100644 --- a/k8s-operator/reconciler/tailscaled/metrics.go +++ b/k8s-operator/reconciler/tailscaled/metrics.go @@ -44,6 +44,7 @@ type MetricsOptions struct { StatefulSetName string // name of the proxy's StatefulSet; not needed by EnsureMetricsResourcesDeleted Namespace string // namespace of the proxy's StatefulSet PodLabels map[string]string // labels that select the proxy's Pods + Labels map[string]string // extra labels for the metrics Service and ServiceMonitor ProxyType string // value of the ts_proxy_type label ParentName string // name of the resource the proxy was created for ParentNamespace string // namespace of the resource the proxy was created for; empty if cluster scoped @@ -274,7 +275,7 @@ func metricsResourceLabels(opts MetricsOptions) map[string]string { lbls[labelPromProxyParentNamespace] = tsoperator.TruncateLabelValue(opts.ParentNamespace) } lbls[labelPromJob] = tsoperator.TruncateLabelValue(promJobName(opts)) - return lbls + return mergeMapKeys(lbls, opts.Labels) } // promJobName constructs the value of the Prometheus job label that will apply to all metrics for a ServiceMonitor. diff --git a/k8s-operator/reconciler/tailscaled/proxyclass.go b/k8s-operator/reconciler/tailscaled/proxyclass.go index cecb93efb..86c5d19ae 100644 --- a/k8s-operator/reconciler/tailscaled/proxyclass.go +++ b/k8s-operator/reconciler/tailscaled/proxyclass.go @@ -17,7 +17,15 @@ import ( // ApplyProxyClass overlays the settings in pc onto ss. It's the generic slice of ProxyClass application used by // any reconciler that produces a tailscaled StatefulSet (peer relay, connector, proxy group, etc.). func ApplyProxyClass(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass, managedLabels, managedAnnotations []string) *appsv1.StatefulSet { - if pc == nil || ss == nil || pc.Spec.StatefulSet == nil { + if pc == nil || ss == nil { + return ss + } + + if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable { + enableMetrics(ss) + } + + if pc.Spec.StatefulSet == nil { return ss } @@ -74,6 +82,19 @@ func ApplyProxyClass(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass, managedLabels return ss } +// enableMetrics makes containerboot serve /metrics alongside /healthz on HealthCheckPort. +func enableMetrics(ss *appsv1.StatefulSet) { + for i := range ss.Spec.Template.Spec.Containers { + c := &ss.Spec.Template.Spec.Containers[i] + if c.Name != containerName { + continue + } + + c.Env = append(c.Env, corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"}) + break + } +} + func mergeProtected(current, custom map[string]string, protected []string) map[string]string { if custom == nil { custom = make(map[string]string) @@ -100,9 +121,13 @@ func applyContainerOverlay(c *corev1.Container, overlay *tsapi.Container) { } for _, e := range overlay.Env { - // Env vars added by ProxyClass are appended; Kubernetes uses the last entry for a duplicate name, so this - // lets the user override anything we set (e.g. TS_USERSPACE) without us having to know the full list. - c.Env = append(c.Env, corev1.EnvVar{Name: string(e.Name), Value: e.Value}) + // Replace rather than append, as server-side apply rejects a StatefulSet with duplicate env names. + env := corev1.EnvVar{Name: string(e.Name), Value: e.Value} + if i := slices.IndexFunc(c.Env, func(v corev1.EnvVar) bool { return v.Name == env.Name }); i >= 0 { + c.Env[i] = env + continue + } + c.Env = append(c.Env, env) } if overlay.Image != "" {