feat(dashboard): route the observability proxy roots from the console

The logs and metrics roots under /apis/cozystack.io and
/apis/apps.cozystack.io are served by the observability proxy, not by
the API server, so the console nginx sends them there ahead of its
/apis prefix location, unbuffered and with the same long timeouts, so
that a live tail reaches the browser as it is written.

nginx resolves a literal upstream once at startup and refuses to start
when the name does not resolve, which would take the console down on
every cluster that does not run the proxy. The upstream is therefore a
variable resolved per request, and such a cluster answers 502 on those
paths only. That resolver does not apply the pod's search domains, so
the config becomes a template the image's entrypoint renders with the
pod's nameservers and the platform cluster domain, which the chart now
passes to the console.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>

hack/console-nginx-header-buffers.bats

@@ -37,7 +37,7 @@ nginx_size_kib() {
     cd "$REPO_ROOT"
 
     conf=$(awk '
-        /^COPY <<.EOF. \/etc\/nginx\/conf\.d\/default\.conf$/ { inblock = 1; next }
+        /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next }
         inblock && /^EOF$/ { inblock = 0; next }
         inblock { print }
     ' "$CONSOLE_CONTAINERFILE")

hack/console-nginx-observability-proxy.bats

@@ -0,0 +1,146 @@
+#!/usr/bin/env bats
+# The console nginx sends the logs and metrics roots of the observability
+# proxy to that proxy instead of the API server. The routing lives in the
+# nginx config baked into the console image, which no chart template and no
+# helm-unittest case can see, so it is pinned here.
+#
+# Three properties are checked. The location regex must take the four path
+# shapes the proxy serves and leave every ordinary Kubernetes API path to the
+# /apis and /api prefix locations; nginx runs the regex locations whenever the
+# longest prefix match is a plain prefix, so the prefix locations must not
+# carry ^~, which would stop that. The location must stream: tail and watch
+# style responses stall behind proxy buffering and die at the default 60s
+# read timeout. And the upstream must be a variable resolved per request,
+# because nginx resolves a literal upstream host once at startup and refuses
+# to start when it does not resolve, which is the case on every cluster that
+# does not run the proxy.
+#
+# The regex is evaluated with perl, whose syntax nginx's PCRE accepts for the
+# constructs used here.
+# Run with: bats hack/console-nginx-observability-proxy.bats
+
+load test_helper
+
+REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
+CONSOLE_CONTAINERFILE="packages/system/dashboard/images/console/Containerfile"
+
+console_nginx_template() {
+    awk '
+        /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next }
+        inblock && /^EOF$/ { inblock = 0; next }
+        inblock { print }
+    ' "$REPO_ROOT/$CONSOLE_CONTAINERFILE"
+}
+
+# Prints the regex of every `location ~ <regex> {` line, one per line.
+regex_locations() {
+    printf '%s\n' "$1" | awk '$1 == "location" && $2 == "~" && $4 == "{" { print $3 }'
+}
+
+# Prints the body of the location block whose regex is $2.
+location_body() {
+    # Through ENVIRON rather than -v, which would expand the backslashes.
+    printf '%s\n' "$1" | LOCATION_RE="$2" awk '
+        $1 == "location" && $2 == "~" && $3 == ENVIRON["LOCATION_RE"] { inblock = 1; next }
+        inblock && $1 == "}" { exit }
+        inblock { print }
+    '
+}
+
+matches() {
+    perl -e 'exit(($ARGV[1] =~ /$ARGV[0]/) ? 0 : 1)' "$1" "$2"
+}
+
+@test "console nginx routes exactly the observability proxy roots to the proxy" {
+    conf=$(console_nginx_template)
+    [ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc in $CONSOLE_CONTAINERFILE" >&2; exit 1; }
+
+    regexes=$(regex_locations "$conf")
+    [ "$(printf '%s\n' "$regexes" | grep -c .)" -eq 1 ] \
+        || { echo "expected exactly one regex location, got: $regexes" >&2; exit 1; }
+    re=$regexes
+
+    for path in \
+        /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query \
+        /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/metrics/api/v1/query_range \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log/select/logsql/tail \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/vminstances/vm1/metrics/api/v1/query
+    do
+        matches "$re" "$path" || { echo "$path does not reach the proxy" >&2; exit 1; }
+    done
+
+    for path in \
+        /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs \
+        /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/workloadmonitors/db \
+        /apis/cozystack.io/v1alpha1/applicationdefinitions/postgres \
+        /apis/cozystack.io/v1alpha1/watch/namespaces/tenant-foo/logs/ \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log/ \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/status \
+        /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log \
+        /apis/apps.cozystack.io/v1alpha1/watch/namespaces/tenant-foo/postgreses/db/log/ \
+        /apis/metrics.k8s.io/v1beta1/namespaces/tenant-foo/pods/ \
+        /api/v1/namespaces/tenant-foo/pods/db-0/log \
+        /k8s/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query
+    do
+        if matches "$re" "$path"; then
+            echo "FAIL: $path would reach the proxy instead of the API server" >&2
+            false
+        fi
+    done
+
+    # A ^~ on the /apis or /api prefix would stop nginx from trying the regex.
+    if printf '%s\n' "$conf" | grep -Eq '^[[:space:]]*location[[:space:]]+\^~'; then
+        echo "FAIL: a ^~ prefix location disables the regex location" >&2
+        false
+    fi
+}
+
+@test "the observability proxy location streams, forwards the token and resolves per request" {
+    conf=$(console_nginx_template)
+    re=$(regex_locations "$conf")
+    body=$(location_body "$conf" "$re")
+    [ -n "$body" ] || { echo "no body found for location ~ $re" >&2; exit 1; }
+
+    for directive in \
+        'proxy_http_version 1.1;' \
+        'proxy_buffering off;' \
+        'proxy_read_timeout 86400s;' \
+        'proxy_send_timeout 86400s;'
+    do
+        printf '%s\n' "$body" | grep -Fqx "        $directive" \
+            || { echo "the proxy location lacks '$directive'" >&2; exit 1; }
+    done
+
+    # nginx forwards client headers unless told otherwise; the proxy needs the
+    # Authorization header the gatekeeper set.
+    if printf '%s\n' "$body" | grep -Eq 'proxy_set_header[[:space:]]+Authorization|proxy_pass_request_headers[[:space:]]+off'; then
+        echo "FAIL: the proxy location drops or rewrites the Authorization header" >&2
+        false
+    fi
+
+    printf '%s\n' "$body" | grep -Eq '^[[:space:]]*resolver[[:space:]]+\$\{NGINX_LOCAL_RESOLVERS\}' \
+        || { echo "the proxy location sets no resolver from the pod's resolv.conf" >&2; exit 1; }
+    printf '%s\n' "$body" | grep -Eq '^[[:space:]]*set[[:space:]]+\$observability_proxy[[:space:]]+observability-proxy\.cozy-observability-proxy\.svc\.\$\{CLUSTER_DOMAIN\}:80;' \
+        || { echo "the proxy location does not name the Service by its fully qualified name" >&2; exit 1; }
+    printf '%s\n' "$body" | grep -Eq '^[[:space:]]*proxy_pass[[:space:]]+http://\$observability_proxy;' \
+        || { echo "the proxy location does not proxy_pass through the variable" >&2; exit 1; }
+}
+
+@test "the console image defines every variable its nginx template substitutes" {
+    cd "$REPO_ROOT"
+    conf=$(console_nginx_template)
+
+    # The image's entrypoint substitutes only variables present in the
+    # environment; one that is not would reach nginx as ${NAME} and fail it.
+    grep -Eq '^ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1$' "$CONSOLE_CONTAINERFILE" \
+        || { echo "the image does not ask its entrypoint for NGINX_LOCAL_RESOLVERS" >&2; exit 1; }
+    grep -Eq '^ENV CLUSTER_DOMAIN=cozy\.local$' "$CONSOLE_CONTAINERFILE" \
+        || { echo "the image has no CLUSTER_DOMAIN default" >&2; exit 1; }
+
+    used=$(printf '%s\n' "$conf" | grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' | sort -u)
+    expected=$(printf '%s\n' '${CLUSTER_DOMAIN}' '${NGINX_LOCAL_RESOLVERS}')
+    [ "$used" = "$expected" ] \
+        || { echo "template substitutes: $used; the image defines: $expected" >&2; exit 1; }
+}

packages/system/dashboard/images/console/Containerfile

@@ -32,7 +32,14 @@ FROM nginxinc/nginx-unprivileged:1.29-alpine@sha256:0c79d56aee561a1d81c63f00eee5
 
 COPY --from=builder /src/apps/console/dist /usr/share/nginx/html
 
-COPY <<'EOF' /etc/nginx/conf.d/default.conf
+# The config is a template the image's entrypoint renders into conf.d at
+# startup, substituting only variables set in the environment: CLUSTER_DOMAIN,
+# which the chart overrides with the platform cluster domain, and
+# NGINX_LOCAL_RESOLVERS, which the entrypoint reads from the pod's resolv.conf.
+ENV CLUSTER_DOMAIN=cozy.local
+ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1
+
+COPY <<'EOF' /etc/nginx/templates/default.conf.template
 map $http_upgrade $connection_upgrade {
     default upgrade;
     '' close;
@@ -54,6 +61,23 @@ server {
     root /usr/share/nginx/html;
     index index.html;
 
+    # Logs and metrics roots of a namespace or of one application, served by
+    # the observability proxy rather than the API server. A regex location wins
+    # over the /apis prefix below. The upstream is a variable so nginx resolves
+    # it per request: on a cluster without the proxy these paths answer 502
+    # instead of nginx refusing to start. The resolver does not apply search
+    # domains, hence the fully qualified name.
+    location ~ ^/apis/(?:cozystack\.io/v1alpha1/namespaces/[^/]+/(?:logs|metrics)|apps\.cozystack\.io/v1alpha1/namespaces/[^/]+/[^/]+/[^/]+/(?:log|metrics))/ {
+        resolver ${NGINX_LOCAL_RESOLVERS};
+        set $observability_proxy observability-proxy.cozy-observability-proxy.svc.${CLUSTER_DOMAIN}:80;
+        proxy_http_version 1.1;
+        proxy_buffering off;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout 86400s;
+        proxy_send_timeout 86400s;
+        proxy_pass http://$observability_proxy;
+    }
+
     # Kubernetes API proxy for both core and aggregated groups.
     # Chunked-encoding watches require proxy_buffering off and a long timeout.
     location /apis {

packages/system/dashboard/templates/console.yaml

@@ -40,6 +40,11 @@ spec:
       containers:
         - image: {{ .Values.console.image | quote }}
           imagePullPolicy: IfNotPresent
+          {{- with index .Values._cluster "cluster-domain" }}
+          env:
+            - name: CLUSTER_DOMAIN
+              value: {{ . | quote }}
+          {{- end }}
           livenessProbe:
             failureThreshold: 3
             httpGet:

packages/system/dashboard/tests/console_cluster_domain_test.yaml

@@ -0,0 +1,42 @@
+suite: console cluster domain
+# The console nginx names the observability proxy by its fully qualified
+# Service name, because the resolver it uses for a per-request upstream does
+# not apply the pod's search domains. The image defaults to cozy.local; the
+# chart passes the platform cluster domain so a cluster on another domain
+# still reaches the proxy.
+templates:
+  - templates/console.yaml
+  - templates/configmap.yaml
+
+release:
+  name: dashboard
+  namespace: cozy-dashboard
+
+tests:
+  - it: passes the platform cluster domain to the console
+    template: templates/console.yaml
+    set:
+      _cluster:
+        cluster-domain: cluster.local
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: CLUSTER_DOMAIN
+            value: cluster.local
+
+  - it: leaves the image default when the platform passes no cluster domain
+    template: templates/console.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.containers[0].env
+
+  # The entrypoint renders the nginx template into /etc/nginx/conf.d at
+  # startup. On a read-only root it logs the failure and starts nginx on the
+  # stock default.conf, which serves no /apis, /api or proxy route at all.
+  - it: keeps the root filesystem writable for the rendered nginx config
+    template: templates/console.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
+          value: false