diff --git a/hack/console-nginx-header-buffers.bats b/hack/console-nginx-header-buffers.bats index f8ce4d949..e7c143e91 100644 --- a/hack/console-nginx-header-buffers.bats +++ b/hack/console-nginx-header-buffers.bats @@ -37,7 +37,7 @@ nginx_size_kib() { cd "$REPO_ROOT" conf=$(awk ' - /^COPY <<.EOF. \/etc\/nginx\/conf\.d\/default\.conf$/ { inblock = 1; next } + /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next } inblock && /^EOF$/ { inblock = 0; next } inblock { print } ' "$CONSOLE_CONTAINERFILE") diff --git a/hack/console-nginx-observability-proxy.bats b/hack/console-nginx-observability-proxy.bats new file mode 100644 index 000000000..0efb0ffcd --- /dev/null +++ b/hack/console-nginx-observability-proxy.bats @@ -0,0 +1,146 @@ +#!/usr/bin/env bats +# The console nginx sends the logs and metrics roots of the observability +# proxy to that proxy instead of the API server. The routing lives in the +# nginx config baked into the console image, which no chart template and no +# helm-unittest case can see, so it is pinned here. +# +# Three properties are checked. The location regex must take the four path +# shapes the proxy serves and leave every ordinary Kubernetes API path to the +# /apis and /api prefix locations; nginx runs the regex locations whenever the +# longest prefix match is a plain prefix, so the prefix locations must not +# carry ^~, which would stop that. The location must stream: tail and watch +# style responses stall behind proxy buffering and die at the default 60s +# read timeout. And the upstream must be a variable resolved per request, +# because nginx resolves a literal upstream host once at startup and refuses +# to start when it does not resolve, which is the case on every cluster that +# does not run the proxy. +# +# The regex is evaluated with perl, whose syntax nginx's PCRE accepts for the +# constructs used here. +# Run with: bats hack/console-nginx-observability-proxy.bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +CONSOLE_CONTAINERFILE="packages/system/dashboard/images/console/Containerfile" + +console_nginx_template() { + awk ' + /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next } + inblock && /^EOF$/ { inblock = 0; next } + inblock { print } + ' "$REPO_ROOT/$CONSOLE_CONTAINERFILE" +} + +# Prints the regex of every `location ~ {` line, one per line. +regex_locations() { + printf '%s\n' "$1" | awk '$1 == "location" && $2 == "~" && $4 == "{" { print $3 }' +} + +# Prints the body of the location block whose regex is $2. +location_body() { + # Through ENVIRON rather than -v, which would expand the backslashes. + printf '%s\n' "$1" | LOCATION_RE="$2" awk ' + $1 == "location" && $2 == "~" && $3 == ENVIRON["LOCATION_RE"] { inblock = 1; next } + inblock && $1 == "}" { exit } + inblock { print } + ' +} + +matches() { + perl -e 'exit(($ARGV[1] =~ /$ARGV[0]/) ? 0 : 1)' "$1" "$2" +} + +@test "console nginx routes exactly the observability proxy roots to the proxy" { + conf=$(console_nginx_template) + [ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc in $CONSOLE_CONTAINERFILE" >&2; exit 1; } + + regexes=$(regex_locations "$conf") + [ "$(printf '%s\n' "$regexes" | grep -c .)" -eq 1 ] \ + || { echo "expected exactly one regex location, got: $regexes" >&2; exit 1; } + re=$regexes + + for path in \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/metrics/api/v1/query_range \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log/select/logsql/tail \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/vminstances/vm1/metrics/api/v1/query + do + matches "$re" "$path" || { echo "$path does not reach the proxy" >&2; exit 1; } + done + + for path in \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/workloadmonitors/db \ + /apis/cozystack.io/v1alpha1/applicationdefinitions/postgres \ + /apis/cozystack.io/v1alpha1/watch/namespaces/tenant-foo/logs/ \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log/ \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/status \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log \ + /apis/apps.cozystack.io/v1alpha1/watch/namespaces/tenant-foo/postgreses/db/log/ \ + /apis/metrics.k8s.io/v1beta1/namespaces/tenant-foo/pods/ \ + /api/v1/namespaces/tenant-foo/pods/db-0/log \ + /k8s/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query + do + if matches "$re" "$path"; then + echo "FAIL: $path would reach the proxy instead of the API server" >&2 + false + fi + done + + # A ^~ on the /apis or /api prefix would stop nginx from trying the regex. + if printf '%s\n' "$conf" | grep -Eq '^[[:space:]]*location[[:space:]]+\^~'; then + echo "FAIL: a ^~ prefix location disables the regex location" >&2 + false + fi +} + +@test "the observability proxy location streams, forwards the token and resolves per request" { + conf=$(console_nginx_template) + re=$(regex_locations "$conf") + body=$(location_body "$conf" "$re") + [ -n "$body" ] || { echo "no body found for location ~ $re" >&2; exit 1; } + + for directive in \ + 'proxy_http_version 1.1;' \ + 'proxy_buffering off;' \ + 'proxy_read_timeout 86400s;' \ + 'proxy_send_timeout 86400s;' + do + printf '%s\n' "$body" | grep -Fqx " $directive" \ + || { echo "the proxy location lacks '$directive'" >&2; exit 1; } + done + + # nginx forwards client headers unless told otherwise; the proxy needs the + # Authorization header the gatekeeper set. + if printf '%s\n' "$body" | grep -Eq 'proxy_set_header[[:space:]]+Authorization|proxy_pass_request_headers[[:space:]]+off'; then + echo "FAIL: the proxy location drops or rewrites the Authorization header" >&2 + false + fi + + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*resolver[[:space:]]+\$\{NGINX_LOCAL_RESOLVERS\}' \ + || { echo "the proxy location sets no resolver from the pod's resolv.conf" >&2; exit 1; } + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*set[[:space:]]+\$observability_proxy[[:space:]]+observability-proxy\.cozy-observability-proxy\.svc\.\$\{CLUSTER_DOMAIN\}:80;' \ + || { echo "the proxy location does not name the Service by its fully qualified name" >&2; exit 1; } + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*proxy_pass[[:space:]]+http://\$observability_proxy;' \ + || { echo "the proxy location does not proxy_pass through the variable" >&2; exit 1; } +} + +@test "the console image defines every variable its nginx template substitutes" { + cd "$REPO_ROOT" + conf=$(console_nginx_template) + + # The image's entrypoint substitutes only variables present in the + # environment; one that is not would reach nginx as ${NAME} and fail it. + grep -Eq '^ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1$' "$CONSOLE_CONTAINERFILE" \ + || { echo "the image does not ask its entrypoint for NGINX_LOCAL_RESOLVERS" >&2; exit 1; } + grep -Eq '^ENV CLUSTER_DOMAIN=cozy\.local$' "$CONSOLE_CONTAINERFILE" \ + || { echo "the image has no CLUSTER_DOMAIN default" >&2; exit 1; } + + used=$(printf '%s\n' "$conf" | grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' | sort -u) + expected=$(printf '%s\n' '${CLUSTER_DOMAIN}' '${NGINX_LOCAL_RESOLVERS}') + [ "$used" = "$expected" ] \ + || { echo "template substitutes: $used; the image defines: $expected" >&2; exit 1; } +} diff --git a/packages/system/dashboard/images/console/Containerfile b/packages/system/dashboard/images/console/Containerfile index 84d634f46..dba07f8d0 100644 --- a/packages/system/dashboard/images/console/Containerfile +++ b/packages/system/dashboard/images/console/Containerfile @@ -32,7 +32,14 @@ FROM nginxinc/nginx-unprivileged:1.29-alpine@sha256:0c79d56aee561a1d81c63f00eee5 COPY --from=builder /src/apps/console/dist /usr/share/nginx/html -COPY <<'EOF' /etc/nginx/conf.d/default.conf +# The config is a template the image's entrypoint renders into conf.d at +# startup, substituting only variables set in the environment: CLUSTER_DOMAIN, +# which the chart overrides with the platform cluster domain, and +# NGINX_LOCAL_RESOLVERS, which the entrypoint reads from the pod's resolv.conf. +ENV CLUSTER_DOMAIN=cozy.local +ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1 + +COPY <<'EOF' /etc/nginx/templates/default.conf.template map $http_upgrade $connection_upgrade { default upgrade; '' close; @@ -54,6 +61,23 @@ server { root /usr/share/nginx/html; index index.html; + # Logs and metrics roots of a namespace or of one application, served by + # the observability proxy rather than the API server. A regex location wins + # over the /apis prefix below. The upstream is a variable so nginx resolves + # it per request: on a cluster without the proxy these paths answer 502 + # instead of nginx refusing to start. The resolver does not apply search + # domains, hence the fully qualified name. + location ~ ^/apis/(?:cozystack\.io/v1alpha1/namespaces/[^/]+/(?:logs|metrics)|apps\.cozystack\.io/v1alpha1/namespaces/[^/]+/[^/]+/[^/]+/(?:log|metrics))/ { + resolver ${NGINX_LOCAL_RESOLVERS}; + set $observability_proxy observability-proxy.cozy-observability-proxy.svc.${CLUSTER_DOMAIN}:80; + proxy_http_version 1.1; + proxy_buffering off; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + proxy_pass http://$observability_proxy; + } + # Kubernetes API proxy for both core and aggregated groups. # Chunked-encoding watches require proxy_buffering off and a long timeout. location /apis { diff --git a/packages/system/dashboard/templates/console.yaml b/packages/system/dashboard/templates/console.yaml index c4e641578..c05f3ee4b 100644 --- a/packages/system/dashboard/templates/console.yaml +++ b/packages/system/dashboard/templates/console.yaml @@ -40,6 +40,11 @@ spec: containers: - image: {{ .Values.console.image | quote }} imagePullPolicy: IfNotPresent + {{- with index .Values._cluster "cluster-domain" }} + env: + - name: CLUSTER_DOMAIN + value: {{ . | quote }} + {{- end }} livenessProbe: failureThreshold: 3 httpGet: diff --git a/packages/system/dashboard/tests/console_cluster_domain_test.yaml b/packages/system/dashboard/tests/console_cluster_domain_test.yaml new file mode 100644 index 000000000..c5e605638 --- /dev/null +++ b/packages/system/dashboard/tests/console_cluster_domain_test.yaml @@ -0,0 +1,42 @@ +suite: console cluster domain +# The console nginx names the observability proxy by its fully qualified +# Service name, because the resolver it uses for a per-request upstream does +# not apply the pod's search domains. The image defaults to cozy.local; the +# chart passes the platform cluster domain so a cluster on another domain +# still reaches the proxy. +templates: + - templates/console.yaml + - templates/configmap.yaml + +release: + name: dashboard + namespace: cozy-dashboard + +tests: + - it: passes the platform cluster domain to the console + template: templates/console.yaml + set: + _cluster: + cluster-domain: cluster.local + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLUSTER_DOMAIN + value: cluster.local + + - it: leaves the image default when the platform passes no cluster domain + template: templates/console.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # The entrypoint renders the nginx template into /etc/nginx/conf.d at + # startup. On a read-only root it logs the failure and starts nginx on the + # stock default.conf, which serves no /apis, /api or proxy route at all. + - it: keeps the root filesystem writable for the rendered nginx config + template: templates/console.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: false