From 47d9ca01f3f936f7ad02571b7a7dbf8fc9e3ec6e Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:37:15 +0700 Subject: [PATCH] feat(observability-proxy): publish the proxy on its own host Clients outside the cluster, such as kubectl, scripts and third-party tools, need to call the proxy with their Kubernetes bearer token. The kube-apiserver service proxy cannot carry it: kube-apiserver drops the Authorization header once it has authenticated the request. The console route sits behind the dashboard gatekeeper, which with OIDC enabled answers an API client that has no session cookie with 401, and exists only where the dashboard itself is published. A route straight to the proxy Service is the path that keeps the token. It stays off until "observability-proxy" is listed in publishing.exposedServices, and follows the dashboard for TLS and for the Ingress versus Gateway API switch. The edge must not end a request the proxy still serves. The tenant ingress-nginx reads upstream with a 10s timeout, while a log tail may stay silent for the proxy's 5m idle limit, so the Ingress waits longer than that. Cilium leaves Envoy's 15s route timeout in place on an HTTPRoute that sets none, which would cut every tail, so the route disables it: the proxy bounds each request on its own. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../templates/httproute.yaml | 28 +++ .../templates/ingress.yaml | 52 +++++ .../tests/publishing_test.yaml | 196 ++++++++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 packages/system/observability-proxy/templates/httproute.yaml create mode 100644 packages/system/observability-proxy/templates/ingress.yaml create mode 100644 packages/system/observability-proxy/tests/publishing_test.yaml diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml new file mode 100644 index 000000000..5d4931031 --- /dev/null +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -0,0 +1,28 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- if and (has "observability-proxy" $exposeServices) (eq $gatewayEnabled "true") }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: observability-proxy +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: {{ $exposeIngress }} + hostnames: + - {{ printf "observability-proxy.%s" $host | quote }} + rules: + - backendRefs: + - name: observability-proxy + port: 80 + {{- /* Without a request timeout Cilium leaves Envoy's 15s route default + in place, which would cut every tail. The proxy bounds each + request itself, so the route sets none. */}} + timeouts: + request: 0s +{{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml new file mode 100644 index 000000000..59e79a32b --- /dev/null +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -0,0 +1,52 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $solver := (index $cluster "solver") | default "http01" }} +{{- $clusterIssuer := (index $cluster "issuer-name") | default "letsencrypt-prod" }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- $wildcardSecret := (index $cluster "wildcard-secret-name") | default "" }} +{{- if and (has "observability-proxy" $exposeServices) (ne $gatewayEnabled "true") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: observability-proxy + annotations: + {{- if not $wildcardSecret }} + cert-manager.io/cluster-issuer: {{ $clusterIssuer }} + {{- if eq $solver "http01" }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + {{- end }} + {{- end }} + {{- /* The proxy refuses a body above 1 MiB itself; nginx's 1m is the + same bound, so an oversized body stops at the edge. */}} + nginx.ingress.kubernetes.io/proxy-body-size: 1m + {{- /* A tail flushes every line; buffered, lines would reach the client + in batches. */}} + nginx.ingress.kubernetes.io/proxy-buffering: "off" + {{- /* The controller default is 10s. The longest silence the proxy + allows is a tail with no new line for --tail-idle-timeout (5m), + after which it ends the response itself. */}} + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" +spec: + ingressClassName: {{ $exposeIngress }} + rules: + - host: observability-proxy.{{ $host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.{{ $host }} + {{- /* Wildcard mode: ingress-nginx serves its default certificate, and + the tls section stays to keep the HTTP-to-HTTPS redirect. */}} + {{- if not $wildcardSecret }} + secretName: observability-proxy-tls + {{- end }} +{{- end }} diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml new file mode 100644 index 000000000..0efc3ca82 --- /dev/null +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -0,0 +1,196 @@ +suite: observability-proxy public endpoint +# Clients outside the cluster reach the proxy on its own host with a +# Kubernetes bearer token, so the route must pass Authorization and the +# /apis/... path through untouched, and must not end a request before the +# proxy does: a non-tail request may take up to --request-timeout (1m), and +# a tail may stay silent up to --tail-idle-timeout (5m). Whole objects are +# compared rather than single keys, so an annotation or a field that should +# not be there fails the test too. +templates: + - templates/ingress.yaml + - templates/httproute.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: publishes nothing without the cluster values + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "false" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing in Gateway mode while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: renders an Ingress with a per-host certificate when gateway is off + set: + _cluster: + root-host: example.org + expose-services: "dashboard,observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "false" + asserts: + - template: templates/httproute.yaml + hasDocuments: + count: 0 + - template: templates/ingress.yaml + isKind: + of: Ingress + - template: templates/ingress.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-ingress-ingressclassname: tenant-public + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec + value: + ingressClassName: tenant-public + rules: + - host: observability-proxy.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.example.org + secretName: observability-proxy-tls + + - it: leaves the http01 ingress class out under the dns01 solver + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + solver: dns01 + issuer-name: letsencrypt-stage + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-stage + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.ingressClassName + value: tenant-root + + - it: relies on the wildcard certificate and keeps the TLS section for the redirect + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + wildcard-secret-name: wildcard-tls + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.tls + value: + - hosts: + - observability-proxy.example.org + + - it: renders an HTTPRoute without a route timeout when gateway is on + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + isKind: + of: HTTPRoute + - template: templates/httproute.yaml + isAPIVersion: + of: gateway.networking.k8s.io/v1 + - template: templates/httproute.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/httproute.yaml + equal: + path: spec + value: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: tenant-public + hostnames: + - observability-proxy.example.org + rules: + - backendRefs: + - name: observability-proxy + port: 80 + timeouts: + request: 0s + + - it: attaches to the tenant-root Gateway when expose-ingress is unset + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "true" + asserts: + - template: templates/httproute.yaml + equal: + path: spec.parentRefs[?(@.name == "cozystack")].namespace + value: tenant-root