mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
feat(observability-proxy): publish the proxy on its own host
Clients outside the cluster, such as kubectl, scripts and third-party tools, need to call the proxy with their Kubernetes bearer token. The kube-apiserver service proxy cannot carry it: kube-apiserver drops the Authorization header once it has authenticated the request. The console route sits behind the dashboard gatekeeper, which with OIDC enabled answers an API client that has no session cookie with 401, and exists only where the dashboard itself is published. A route straight to the proxy Service is the path that keeps the token. It stays off until "observability-proxy" is listed in publishing.exposedServices, and follows the dashboard for TLS and for the Ingress versus Gateway API switch. The edge must not end a request the proxy still serves. The tenant ingress-nginx reads upstream with a 10s timeout, while a log tail may stay silent for the proxy's 5m idle limit, so the Ingress waits longer than that. Cilium leaves Envoy's 15s route timeout in place on an HTTPRoute that sets none, which would cut every tail, so the route disables it: the proxy bounds each request on its own. Assisted-by: LLM Signed-off-by: Maxim Belyy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
59cbf4b28e
commit
47d9ca01f3
3 files changed
+276
No files matched your search
@@ -0,0 +1,28 @@
|
||||
{{- $cluster := .Values._cluster | default dict }}
|
||||
{{- $host := index $cluster "root-host" }}
|
||||
{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }}
|
||||
{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }}
|
||||
{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }}
|
||||
{{- if and (has "observability-proxy" $exposeServices) (eq $gatewayEnabled "true") }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: observability-proxy
|
||||
spec:
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: cozystack
|
||||
namespace: {{ $exposeIngress }}
|
||||
hostnames:
|
||||
- {{ printf "observability-proxy.%s" $host | quote }}
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: observability-proxy
|
||||
port: 80
|
||||
{{- /* Without a request timeout Cilium leaves Envoy's 15s route default
|
||||
in place, which would cut every tail. The proxy bounds each
|
||||
request itself, so the route sets none. */}}
|
||||
timeouts:
|
||||
request: 0s
|
||||
{{- end }}
|
||||
@@ -0,0 +1,52 @@
|
||||
{{- $cluster := .Values._cluster | default dict }}
|
||||
{{- $solver := (index $cluster "solver") | default "http01" }}
|
||||
{{- $clusterIssuer := (index $cluster "issuer-name") | default "letsencrypt-prod" }}
|
||||
{{- $host := index $cluster "root-host" }}
|
||||
{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }}
|
||||
{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }}
|
||||
{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }}
|
||||
{{- $wildcardSecret := (index $cluster "wildcard-secret-name") | default "" }}
|
||||
{{- if and (has "observability-proxy" $exposeServices) (ne $gatewayEnabled "true") }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: observability-proxy
|
||||
annotations:
|
||||
{{- if not $wildcardSecret }}
|
||||
cert-manager.io/cluster-issuer: {{ $clusterIssuer }}
|
||||
{{- if eq $solver "http01" }}
|
||||
acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- /* The proxy refuses a body above 1 MiB itself; nginx's 1m is the
|
||||
same bound, so an oversized body stops at the edge. */}}
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: 1m
|
||||
{{- /* A tail flushes every line; buffered, lines would reach the client
|
||||
in batches. */}}
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
{{- /* The controller default is 10s. The longest silence the proxy
|
||||
allows is a tail with no new line for --tail-idle-timeout (5m),
|
||||
after which it ends the response itself. */}}
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "360"
|
||||
spec:
|
||||
ingressClassName: {{ $exposeIngress }}
|
||||
rules:
|
||||
- host: observability-proxy.{{ $host }}
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: observability-proxy
|
||||
port:
|
||||
name: http
|
||||
tls:
|
||||
- hosts:
|
||||
- observability-proxy.{{ $host }}
|
||||
{{- /* Wildcard mode: ingress-nginx serves its default certificate, and
|
||||
the tls section stays to keep the HTTP-to-HTTPS redirect. */}}
|
||||
{{- if not $wildcardSecret }}
|
||||
secretName: observability-proxy-tls
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,196 @@
|
||||
suite: observability-proxy public endpoint
|
||||
# Clients outside the cluster reach the proxy on its own host with a
|
||||
# Kubernetes bearer token, so the route must pass Authorization and the
|
||||
# /apis/... path through untouched, and must not end a request before the
|
||||
# proxy does: a non-tail request may take up to --request-timeout (1m), and
|
||||
# a tail may stay silent up to --tail-idle-timeout (5m). Whole objects are
|
||||
# compared rather than single keys, so an annotation or a field that should
|
||||
# not be there fails the test too.
|
||||
templates:
|
||||
- templates/ingress.yaml
|
||||
- templates/httproute.yaml
|
||||
|
||||
release:
|
||||
name: observability-proxy
|
||||
namespace: cozy-observability-proxy
|
||||
|
||||
tests:
|
||||
- it: publishes nothing without the cluster values
|
||||
asserts:
|
||||
- template: templates/ingress.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
- template: templates/httproute.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: publishes nothing while observability-proxy is not in expose-services
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "api,dashboard"
|
||||
gateway-enabled: "false"
|
||||
asserts:
|
||||
- template: templates/ingress.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
- template: templates/httproute.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: publishes nothing in Gateway mode while observability-proxy is not in expose-services
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "api,dashboard"
|
||||
gateway-enabled: "true"
|
||||
asserts:
|
||||
- template: templates/ingress.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
- template: templates/httproute.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: renders an Ingress with a per-host certificate when gateway is off
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "dashboard,observability-proxy"
|
||||
expose-ingress: tenant-public
|
||||
gateway-enabled: "false"
|
||||
asserts:
|
||||
- template: templates/httproute.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
- template: templates/ingress.yaml
|
||||
isKind:
|
||||
of: Ingress
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: metadata.name
|
||||
value: observability-proxy
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: metadata.annotations
|
||||
value:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
acme.cert-manager.io/http01-ingress-ingressclassname: tenant-public
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: 1m
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "360"
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: spec
|
||||
value:
|
||||
ingressClassName: tenant-public
|
||||
rules:
|
||||
- host: observability-proxy.example.org
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: observability-proxy
|
||||
port:
|
||||
name: http
|
||||
tls:
|
||||
- hosts:
|
||||
- observability-proxy.example.org
|
||||
secretName: observability-proxy-tls
|
||||
|
||||
- it: leaves the http01 ingress class out under the dns01 solver
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "observability-proxy"
|
||||
gateway-enabled: "false"
|
||||
solver: dns01
|
||||
issuer-name: letsencrypt-stage
|
||||
asserts:
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: metadata.annotations
|
||||
value:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-stage
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: 1m
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "360"
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: spec.ingressClassName
|
||||
value: tenant-root
|
||||
|
||||
- it: relies on the wildcard certificate and keeps the TLS section for the redirect
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "observability-proxy"
|
||||
gateway-enabled: "false"
|
||||
wildcard-secret-name: wildcard-tls
|
||||
asserts:
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: metadata.annotations
|
||||
value:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: 1m
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "360"
|
||||
- template: templates/ingress.yaml
|
||||
equal:
|
||||
path: spec.tls
|
||||
value:
|
||||
- hosts:
|
||||
- observability-proxy.example.org
|
||||
|
||||
- it: renders an HTTPRoute without a route timeout when gateway is on
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "observability-proxy"
|
||||
expose-ingress: tenant-public
|
||||
gateway-enabled: "true"
|
||||
asserts:
|
||||
- template: templates/ingress.yaml
|
||||
hasDocuments:
|
||||
count: 0
|
||||
- template: templates/httproute.yaml
|
||||
isKind:
|
||||
of: HTTPRoute
|
||||
- template: templates/httproute.yaml
|
||||
isAPIVersion:
|
||||
of: gateway.networking.k8s.io/v1
|
||||
- template: templates/httproute.yaml
|
||||
equal:
|
||||
path: metadata.name
|
||||
value: observability-proxy
|
||||
- template: templates/httproute.yaml
|
||||
equal:
|
||||
path: spec
|
||||
value:
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: cozystack
|
||||
namespace: tenant-public
|
||||
hostnames:
|
||||
- observability-proxy.example.org
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: observability-proxy
|
||||
port: 80
|
||||
timeouts:
|
||||
request: 0s
|
||||
|
||||
- it: attaches to the tenant-root Gateway when expose-ingress is unset
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
expose-services: "observability-proxy"
|
||||
gateway-enabled: "true"
|
||||
asserts:
|
||||
- template: templates/httproute.yaml
|
||||
equal:
|
||||
path: spec.parentRefs[?(@.name == "cozystack")].namespace
|
||||
value: tenant-root
|
||||
Reference in new issue
Block a user