From 47d9ca01f3f936f7ad02571b7a7dbf8fc9e3ec6e Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:37:15 +0700 Subject: [PATCH 1/4] feat(observability-proxy): publish the proxy on its own host Clients outside the cluster, such as kubectl, scripts and third-party tools, need to call the proxy with their Kubernetes bearer token. The kube-apiserver service proxy cannot carry it: kube-apiserver drops the Authorization header once it has authenticated the request. The console route sits behind the dashboard gatekeeper, which with OIDC enabled answers an API client that has no session cookie with 401, and exists only where the dashboard itself is published. A route straight to the proxy Service is the path that keeps the token. It stays off until "observability-proxy" is listed in publishing.exposedServices, and follows the dashboard for TLS and for the Ingress versus Gateway API switch. The edge must not end a request the proxy still serves. The tenant ingress-nginx reads upstream with a 10s timeout, while a log tail may stay silent for the proxy's 5m idle limit, so the Ingress waits longer than that. Cilium leaves Envoy's 15s route timeout in place on an HTTPRoute that sets none, which would cut every tail, so the route disables it: the proxy bounds each request on its own. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../templates/httproute.yaml | 28 +++ .../templates/ingress.yaml | 52 +++++ .../tests/publishing_test.yaml | 196 ++++++++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 packages/system/observability-proxy/templates/httproute.yaml create mode 100644 packages/system/observability-proxy/templates/ingress.yaml create mode 100644 packages/system/observability-proxy/tests/publishing_test.yaml diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml new file mode 100644 index 000000000..5d4931031 --- /dev/null +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -0,0 +1,28 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- if and (has "observability-proxy" $exposeServices) (eq $gatewayEnabled "true") }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: observability-proxy +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: {{ $exposeIngress }} + hostnames: + - {{ printf "observability-proxy.%s" $host | quote }} + rules: + - backendRefs: + - name: observability-proxy + port: 80 + {{- /* Without a request timeout Cilium leaves Envoy's 15s route default + in place, which would cut every tail. The proxy bounds each + request itself, so the route sets none. */}} + timeouts: + request: 0s +{{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml new file mode 100644 index 000000000..59e79a32b --- /dev/null +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -0,0 +1,52 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $solver := (index $cluster "solver") | default "http01" }} +{{- $clusterIssuer := (index $cluster "issuer-name") | default "letsencrypt-prod" }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- $wildcardSecret := (index $cluster "wildcard-secret-name") | default "" }} +{{- if and (has "observability-proxy" $exposeServices) (ne $gatewayEnabled "true") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: observability-proxy + annotations: + {{- if not $wildcardSecret }} + cert-manager.io/cluster-issuer: {{ $clusterIssuer }} + {{- if eq $solver "http01" }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + {{- end }} + {{- end }} + {{- /* The proxy refuses a body above 1 MiB itself; nginx's 1m is the + same bound, so an oversized body stops at the edge. */}} + nginx.ingress.kubernetes.io/proxy-body-size: 1m + {{- /* A tail flushes every line; buffered, lines would reach the client + in batches. */}} + nginx.ingress.kubernetes.io/proxy-buffering: "off" + {{- /* The controller default is 10s. The longest silence the proxy + allows is a tail with no new line for --tail-idle-timeout (5m), + after which it ends the response itself. */}} + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" +spec: + ingressClassName: {{ $exposeIngress }} + rules: + - host: observability-proxy.{{ $host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.{{ $host }} + {{- /* Wildcard mode: ingress-nginx serves its default certificate, and + the tls section stays to keep the HTTP-to-HTTPS redirect. */}} + {{- if not $wildcardSecret }} + secretName: observability-proxy-tls + {{- end }} +{{- end }} diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml new file mode 100644 index 000000000..0efc3ca82 --- /dev/null +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -0,0 +1,196 @@ +suite: observability-proxy public endpoint +# Clients outside the cluster reach the proxy on its own host with a +# Kubernetes bearer token, so the route must pass Authorization and the +# /apis/... path through untouched, and must not end a request before the +# proxy does: a non-tail request may take up to --request-timeout (1m), and +# a tail may stay silent up to --tail-idle-timeout (5m). Whole objects are +# compared rather than single keys, so an annotation or a field that should +# not be there fails the test too. +templates: + - templates/ingress.yaml + - templates/httproute.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: publishes nothing without the cluster values + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "false" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing in Gateway mode while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: renders an Ingress with a per-host certificate when gateway is off + set: + _cluster: + root-host: example.org + expose-services: "dashboard,observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "false" + asserts: + - template: templates/httproute.yaml + hasDocuments: + count: 0 + - template: templates/ingress.yaml + isKind: + of: Ingress + - template: templates/ingress.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-ingress-ingressclassname: tenant-public + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec + value: + ingressClassName: tenant-public + rules: + - host: observability-proxy.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.example.org + secretName: observability-proxy-tls + + - it: leaves the http01 ingress class out under the dns01 solver + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + solver: dns01 + issuer-name: letsencrypt-stage + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-stage + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.ingressClassName + value: tenant-root + + - it: relies on the wildcard certificate and keeps the TLS section for the redirect + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + wildcard-secret-name: wildcard-tls + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.tls + value: + - hosts: + - observability-proxy.example.org + + - it: renders an HTTPRoute without a route timeout when gateway is on + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + isKind: + of: HTTPRoute + - template: templates/httproute.yaml + isAPIVersion: + of: gateway.networking.k8s.io/v1 + - template: templates/httproute.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/httproute.yaml + equal: + path: spec + value: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: tenant-public + hostnames: + - observability-proxy.example.org + rules: + - backendRefs: + - name: observability-proxy + port: 80 + timeouts: + request: 0s + + - it: attaches to the tenant-root Gateway when expose-ingress is unset + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "true" + asserts: + - template: templates/httproute.yaml + equal: + path: spec.parentRefs[?(@.name == "cozystack")].namespace + value: tenant-root From 77acd23bdda7a3f514a2b090aa2a5d5096c38a86 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:38:18 +0700 Subject: [PATCH 2/4] feat(platform): let the observability proxy attach to the tenant Gateway With the Gateway API enabled, the publishing Gateway admits routes only from the publishing tenant and the namespaces in gateway.attachedNamespaces. The observability proxy publishes its HTTPRoute from its own namespace, so without an entry the route renders and is never served. Clusters that override gateway.attachedNamespaces replace the whole list and have to add the namespace themselves. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../tests/observability_proxy_wiring_test.yaml | 10 ++++++++++ packages/core/platform/values.yaml | 1 + 2 files changed, 11 insertions(+) diff --git a/packages/core/platform/tests/observability_proxy_wiring_test.yaml b/packages/core/platform/tests/observability_proxy_wiring_test.yaml index 9e6ec583d..84777ca7f 100644 --- a/packages/core/platform/tests/observability_proxy_wiring_test.yaml +++ b/packages/core/platform/tests/observability_proxy_wiring_test.yaml @@ -5,6 +5,7 @@ suite: observability-proxy package wiring templates: - templates/sources.yaml - templates/bundles/system.yaml + - templates/apps.yaml release: name: cozystack namespace: cozy-system @@ -67,3 +68,12 @@ tests: kind: Package name: cozystack.observability-proxy any: true + + # The proxy's HTTPRoute lives in its own namespace, and the tenant Gateway + # admits routes only from the namespaces listed here. + - it: lets the proxy namespace attach a route to the publishing Gateway + template: templates/apps.yaml + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: 'gateway-attached-namespaces: "([^"]*,)?cozy-observability-proxy(,[^"]*)?"' diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f2ce252ed..5ca66ac17 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -668,6 +668,7 @@ gateway: - cozy-kubevirt-cdi - cozy-monitoring - cozy-linstor-gui + - cozy-observability-proxy - default # Platform monitoring configuration. monitoring: From 773a3a96d3d7a378914c72823857580f3920d8b7 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:42:46 +0700 Subject: [PATCH 3/4] feat(tenant): allow tenant egress to the observability proxy The observability proxy's Ingress is served by the ingress controller in the publishing tenant, and tenant egress is denied except for the allow-to-* policies. Without one for the proxy namespace, every request through the Ingress and every HTTP-01 challenge for its certificate times out at the controller. The policy matches the whole namespace because cert-manager runs the HTTP-01 solver pods there, next to the proxy. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../apps/tenant/templates/networkpolicy.yaml | 14 ++++ ...etworkpolicy_observability_proxy_test.yaml | 64 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml diff --git a/packages/apps/tenant/templates/networkpolicy.yaml b/packages/apps/tenant/templates/networkpolicy.yaml index 5285b495a..bc6904e0a 100644 --- a/packages/apps/tenant/templates/networkpolicy.yaml +++ b/packages/apps/tenant/templates/networkpolicy.yaml @@ -259,6 +259,20 @@ spec: - ports: - port: "8443" protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + # The whole namespace, as for linstor-gui: the ingress controller also + # has to reach the ACME HTTP-01 solver pods cert-manager starts there. + name: allow-to-observability-proxy + namespace: {{ include "tenant.name" . }} +spec: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy {{- if .Values.monitoring }} --- apiVersion: cilium.io/v2 diff --git a/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml new file mode 100644 index 000000000..3f3e027fd --- /dev/null +++ b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml @@ -0,0 +1,64 @@ +suite: tenant egress to the observability proxy +# The observability-proxy Ingress in cozy-observability-proxy is served by +# the ingress controller running in tenant-root. Tenant egress is +# default-deny plus explicit allow-to-* policies, so without this one the +# controller reaches neither the proxy nor the ACME HTTP-01 solver pod that +# cert-manager starts in the same namespace; hence the whole namespace, as +# allow-to-linstor-gui does. +templates: + - templates/networkpolicy.yaml +tests: + - it: lets every pod of the root tenant reach the proxy namespace + release: + name: tenant-root + namespace: tenant-root + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + isKind: + of: CiliumNetworkPolicy + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-root + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: spec + value: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy + + - it: renders in a nested tenant with every module off + release: + name: tenant-bar + namespace: tenant-foo + set: + monitoring: false + ingress: false + etcd: false + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-foo-bar + + - it: renders nothing where the tenant Cilium policies are off + release: + name: tenant-root + namespace: tenant-root + set: + _cluster: + bundle-name: isp-hosted + asserts: + - hasDocuments: + count: 0 From d21d9de641db88f2e9fec6fbbe45d1d8500dc5a4 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:43:31 +0700 Subject: [PATCH 4/4] fix(observability-proxy): end silent tails before the Gateway does Behind a Cilium Gateway, Envoy resets a stream that carries no data for 300s, the default stream idle timeout, and the proxy's own tail idle limit was the same 5m. A tail that went quiet could then be cut by Envoy rather than ended by the proxy, which the client sees as a reset stream instead of a finished response. The HTTPRoute cannot raise Envoy's limit, so the proxy ends a silent tail a little earlier. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../system/observability-proxy/templates/deployment.yaml | 4 ++++ .../system/observability-proxy/templates/httproute.yaml | 4 +++- .../system/observability-proxy/templates/ingress.yaml | 5 +++-- .../observability-proxy/tests/deployment_test.yaml | 9 +++++++++ .../observability-proxy/tests/publishing_test.yaml | 2 +- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index 581c15fde..dd95dfbb6 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -38,6 +38,10 @@ spec: - --cluster-domain={{ $clusterDomain }} - --root-tenant={{ .Values.observabilityProxy.rootTenant }} - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + {{- /* Below the 300s stream idle timeout of a Cilium Gateway, so a + published tail that stops sending lines is ended by the proxy, + not reset by Envoy. */}} + - --tail-idle-timeout=4m30s ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml index 5d4931031..1ddd2f720 100644 --- a/packages/system/observability-proxy/templates/httproute.yaml +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -22,7 +22,9 @@ spec: port: 80 {{- /* Without a request timeout Cilium leaves Envoy's 15s route default in place, which would cut every tail. The proxy bounds each - request itself, so the route sets none. */}} + request itself, so the route disables it. Envoy's 300s stream + idle timeout still applies; the proxy's tail idle limit stays + below it (see deployment.yaml). */}} timeouts: request: 0s {{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml index 59e79a32b..7c6b1a029 100644 --- a/packages/system/observability-proxy/templates/ingress.yaml +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -25,8 +25,9 @@ metadata: in batches. */}} nginx.ingress.kubernetes.io/proxy-buffering: "off" {{- /* The controller default is 10s. The longest silence the proxy - allows is a tail with no new line for --tail-idle-timeout (5m), - after which it ends the response itself. */}} + allows is a tail with no new line for --tail-idle-timeout (4m30s), + after which it ends the response itself; before the first byte + the token and access reviews may add up to --request-timeout (1m). */}} nginx.ingress.kubernetes.io/proxy-read-timeout: "360" spec: ingressClassName: {{ $exposeIngress }} diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index ddf9d34b8..978f9e968 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -40,6 +40,15 @@ tests: path: spec.template.spec.containers[0].args content: --auth-cache-ttl=10s + # Envoy behind a Cilium Gateway resets a stream after 300s without data + # (envoy.streamIdleTimeoutDurationSeconds), so a silent tail has to be + # ended by the proxy first, or the client sees a reset instead of an end. + - it: ends a silent tail before the Gateway stream idle timeout + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --tail-idle-timeout=4m30s + - it: falls back to cozy.local when the platform passes no cluster domain asserts: - contains: diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml index 0efc3ca82..676c38e08 100644 --- a/packages/system/observability-proxy/tests/publishing_test.yaml +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -3,7 +3,7 @@ suite: observability-proxy public endpoint # Kubernetes bearer token, so the route must pass Authorization and the # /apis/... path through untouched, and must not end a request before the # proxy does: a non-tail request may take up to --request-timeout (1m), and -# a tail may stay silent up to --tail-idle-timeout (5m). Whole objects are +# a tail may stay silent up to --tail-idle-timeout (4m30s). Whole objects are # compared rather than single keys, so an annotation or a field that should # not be there fails the test too. templates: