diff --git a/packages/apps/tenant/templates/networkpolicy.yaml b/packages/apps/tenant/templates/networkpolicy.yaml index 5285b495a..bc6904e0a 100644 --- a/packages/apps/tenant/templates/networkpolicy.yaml +++ b/packages/apps/tenant/templates/networkpolicy.yaml @@ -259,6 +259,20 @@ spec: - ports: - port: "8443" protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + # The whole namespace, as for linstor-gui: the ingress controller also + # has to reach the ACME HTTP-01 solver pods cert-manager starts there. + name: allow-to-observability-proxy + namespace: {{ include "tenant.name" . }} +spec: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy {{- if .Values.monitoring }} --- apiVersion: cilium.io/v2 diff --git a/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml new file mode 100644 index 000000000..3f3e027fd --- /dev/null +++ b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml @@ -0,0 +1,64 @@ +suite: tenant egress to the observability proxy +# The observability-proxy Ingress in cozy-observability-proxy is served by +# the ingress controller running in tenant-root. Tenant egress is +# default-deny plus explicit allow-to-* policies, so without this one the +# controller reaches neither the proxy nor the ACME HTTP-01 solver pod that +# cert-manager starts in the same namespace; hence the whole namespace, as +# allow-to-linstor-gui does. +templates: + - templates/networkpolicy.yaml +tests: + - it: lets every pod of the root tenant reach the proxy namespace + release: + name: tenant-root + namespace: tenant-root + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + isKind: + of: CiliumNetworkPolicy + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-root + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: spec + value: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy + + - it: renders in a nested tenant with every module off + release: + name: tenant-bar + namespace: tenant-foo + set: + monitoring: false + ingress: false + etcd: false + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-foo-bar + + - it: renders nothing where the tenant Cilium policies are off + release: + name: tenant-root + namespace: tenant-root + set: + _cluster: + bundle-name: isp-hosted + asserts: + - hasDocuments: + count: 0 diff --git a/packages/core/platform/tests/observability_proxy_wiring_test.yaml b/packages/core/platform/tests/observability_proxy_wiring_test.yaml index 9e6ec583d..84777ca7f 100644 --- a/packages/core/platform/tests/observability_proxy_wiring_test.yaml +++ b/packages/core/platform/tests/observability_proxy_wiring_test.yaml @@ -5,6 +5,7 @@ suite: observability-proxy package wiring templates: - templates/sources.yaml - templates/bundles/system.yaml + - templates/apps.yaml release: name: cozystack namespace: cozy-system @@ -67,3 +68,12 @@ tests: kind: Package name: cozystack.observability-proxy any: true + + # The proxy's HTTPRoute lives in its own namespace, and the tenant Gateway + # admits routes only from the namespaces listed here. + - it: lets the proxy namespace attach a route to the publishing Gateway + template: templates/apps.yaml + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: 'gateway-attached-namespaces: "([^"]*,)?cozy-observability-proxy(,[^"]*)?"' diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f2e35b5e5..e5a488498 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -668,6 +668,7 @@ gateway: - cozy-kubevirt-cdi - cozy-monitoring - cozy-linstor-gui + - cozy-observability-proxy - default # Platform ingress configuration. ingress: diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index d49de140b..1fa696797 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -38,6 +38,10 @@ spec: - --cluster-domain={{ $clusterDomain }} - --root-tenant={{ .Values.observabilityProxy.rootTenant }} - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + {{- /* Below the 300s stream idle timeout of a Cilium Gateway, so a + published tail that stops sending lines is ended by the proxy, + not reset by Envoy. */}} + - --tail-idle-timeout=4m30s - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml new file mode 100644 index 000000000..1ddd2f720 --- /dev/null +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -0,0 +1,30 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- if and (has "observability-proxy" $exposeServices) (eq $gatewayEnabled "true") }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: observability-proxy +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: {{ $exposeIngress }} + hostnames: + - {{ printf "observability-proxy.%s" $host | quote }} + rules: + - backendRefs: + - name: observability-proxy + port: 80 + {{- /* Without a request timeout Cilium leaves Envoy's 15s route default + in place, which would cut every tail. The proxy bounds each + request itself, so the route disables it. Envoy's 300s stream + idle timeout still applies; the proxy's tail idle limit stays + below it (see deployment.yaml). */}} + timeouts: + request: 0s +{{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml new file mode 100644 index 000000000..7c6b1a029 --- /dev/null +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -0,0 +1,53 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $solver := (index $cluster "solver") | default "http01" }} +{{- $clusterIssuer := (index $cluster "issuer-name") | default "letsencrypt-prod" }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- $wildcardSecret := (index $cluster "wildcard-secret-name") | default "" }} +{{- if and (has "observability-proxy" $exposeServices) (ne $gatewayEnabled "true") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: observability-proxy + annotations: + {{- if not $wildcardSecret }} + cert-manager.io/cluster-issuer: {{ $clusterIssuer }} + {{- if eq $solver "http01" }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + {{- end }} + {{- end }} + {{- /* The proxy refuses a body above 1 MiB itself; nginx's 1m is the + same bound, so an oversized body stops at the edge. */}} + nginx.ingress.kubernetes.io/proxy-body-size: 1m + {{- /* A tail flushes every line; buffered, lines would reach the client + in batches. */}} + nginx.ingress.kubernetes.io/proxy-buffering: "off" + {{- /* The controller default is 10s. The longest silence the proxy + allows is a tail with no new line for --tail-idle-timeout (4m30s), + after which it ends the response itself; before the first byte + the token and access reviews may add up to --request-timeout (1m). */}} + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" +spec: + ingressClassName: {{ $exposeIngress }} + rules: + - host: observability-proxy.{{ $host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.{{ $host }} + {{- /* Wildcard mode: ingress-nginx serves its default certificate, and + the tls section stays to keep the HTTP-to-HTTPS redirect. */}} + {{- if not $wildcardSecret }} + secretName: observability-proxy-tls + {{- end }} +{{- end }} diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index a5e948920..ef95d75f7 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -40,6 +40,15 @@ tests: path: spec.template.spec.containers[0].args content: --auth-cache-ttl=10s + # Envoy behind a Cilium Gateway resets a stream after 300s without data + # (envoy.streamIdleTimeoutDurationSeconds), so a silent tail has to be + # ended by the proxy first, or the client sees a reset instead of an end. + - it: ends a silent tail before the Gateway stream idle timeout + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --tail-idle-timeout=4m30s + - it: passes the metrics store names and short-term retention asserts: - contains: diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml new file mode 100644 index 000000000..676c38e08 --- /dev/null +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -0,0 +1,196 @@ +suite: observability-proxy public endpoint +# Clients outside the cluster reach the proxy on its own host with a +# Kubernetes bearer token, so the route must pass Authorization and the +# /apis/... path through untouched, and must not end a request before the +# proxy does: a non-tail request may take up to --request-timeout (1m), and +# a tail may stay silent up to --tail-idle-timeout (4m30s). Whole objects are +# compared rather than single keys, so an annotation or a field that should +# not be there fails the test too. +templates: + - templates/ingress.yaml + - templates/httproute.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: publishes nothing without the cluster values + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "false" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing in Gateway mode while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: renders an Ingress with a per-host certificate when gateway is off + set: + _cluster: + root-host: example.org + expose-services: "dashboard,observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "false" + asserts: + - template: templates/httproute.yaml + hasDocuments: + count: 0 + - template: templates/ingress.yaml + isKind: + of: Ingress + - template: templates/ingress.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-ingress-ingressclassname: tenant-public + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec + value: + ingressClassName: tenant-public + rules: + - host: observability-proxy.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.example.org + secretName: observability-proxy-tls + + - it: leaves the http01 ingress class out under the dns01 solver + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + solver: dns01 + issuer-name: letsencrypt-stage + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-stage + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.ingressClassName + value: tenant-root + + - it: relies on the wildcard certificate and keeps the TLS section for the redirect + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + wildcard-secret-name: wildcard-tls + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.tls + value: + - hosts: + - observability-proxy.example.org + + - it: renders an HTTPRoute without a route timeout when gateway is on + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + isKind: + of: HTTPRoute + - template: templates/httproute.yaml + isAPIVersion: + of: gateway.networking.k8s.io/v1 + - template: templates/httproute.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/httproute.yaml + equal: + path: spec + value: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: tenant-public + hostnames: + - observability-proxy.example.org + rules: + - backendRefs: + - name: observability-proxy + port: 80 + timeouts: + request: 0s + + - it: attaches to the tenant-root Gateway when expose-ingress is unset + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "true" + asserts: + - template: templates/httproute.yaml + equal: + path: spec.parentRefs[?(@.name == "cozystack")].namespace + value: tenant-root