fix(kubernetes, monitoring): send tenant cluster metrics to the tenant's vminsert

The monitoring-agents chart reads vmagent.remoteWrite.urls as a list and
takes the tenant label from global.target, while the kubernetes app still
passed a single remoteWrite.url and vmagent.externalLabels.tenant. Both
keys were dropped in silence, so the vmagent of every tenant cluster fell
back to the chart defaults: it wrote to vminsert services in
cozy-monitoring that do not exist inside the tenant cluster, queued the
failed writes on the node disk, and labelled every series
tenant="cozy-monitoring". No tenant cluster metric reached the tenant's
Monitoring.

The tenant label has to name the tenant that owns the cluster, which is
not always the tenant whose Monitoring receives the data: a tenant
without its own Monitoring inherits a parent's.

Assisted-by: LLM
Signed-off-by: Mattia Eleuteri <mattia@hidora.io>

packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml

@@ -45,7 +45,8 @@ spec:
         cluster: {{ .Release.Name }}
         tenant: {{ .Release.Namespace }}
       remoteWrite:
-        url: http://vminsert-shortterm.{{ $targetTenant }}.svc.{{ $clusterDomain }}:8480/insert/0/prometheus
+        urls:
+          - http://vminsert-shortterm.{{ $targetTenant }}.svc.{{ $clusterDomain }}:8480/insert/0/prometheus
     fluent-bit:
       readinessProbe:
         httpGet:

packages/apps/kubernetes/tests/monitoring_agents_test.yaml

@@ -0,0 +1,42 @@
+suite: monitoring-agents HelmRelease values match the monitoring-agents chart
+
+# The values below are consumed by packages/system/monitoring-agents, which
+# reads vmagent.remoteWrite.urls as a list and the tenant label from
+# vmagent.externalLabels.tenant. A key the chart does not read is dropped in
+# silence, and the chart then falls back to cozy-monitoring defaults that do
+# not resolve inside a tenant cluster. Its counterpart is
+# packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml.
+templates:
+  - templates/helmreleases/monitoring-agents.yaml
+
+values:
+  - values/common.yaml
+
+release:
+  name: kubernetes-clusterprod
+  namespace: tenant-a
+
+set:
+  addons.monitoringAgents.enabled: true
+  _namespace.monitoring: tenant-root
+  _cluster:
+    cluster-domain: cozy.local
+
+tests:
+  - it: sends metrics to the shortterm vminsert of the monitoring tenant as a urls list
+    asserts:
+      - equal:
+          path: spec.values.vmagent.remoteWrite
+          value:
+            urls:
+              - http://vminsert-shortterm.tenant-root.svc.cozy.local:8480/insert/0/prometheus
+
+  - it: labels series with the tenant that owns the cluster, not the monitoring tenant
+    asserts:
+      - equal:
+          path: spec.values.vmagent.externalLabels
+          value:
+            cluster: kubernetes-clusterprod
+            tenant: tenant-a
+      - notExists:
+          path: spec.values.global

packages/system/monitoring-agents/templates/vmagent.yaml

@@ -10,7 +10,7 @@ spec:
   shardCount: 1
   externalLabels:
     cluster: {{ .Values.vmagent.externalLabels.cluster }}
-    tenant: {{ .Values.global.target }}
+    tenant: {{ .Values.vmagent.externalLabels.tenant | default .Values.global.target | quote }}
   extraArgs:
     {{- toYaml (deepCopy .Values.vmagent.extraArgs | mergeOverwrite (fromYaml (include "monitoring-agents.vmagent.defaultExtraArgs" .))) | nindent 4 }}
   remoteWrite:

packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml

@@ -0,0 +1,65 @@
+suite: vmagent remote write targets and tenant label
+
+# The root bundle sets only global.target, and the chart defaults derive both
+# the remote write URLs and the tenant label from it. A tenant Kubernetes
+# cluster gets this chart through packages/apps/kubernetes instead, which
+# passes vmagent.remoteWrite.urls and vmagent.externalLabels.tenant and leaves
+# global.target at its default. If either value is dropped, vmagent writes to
+# cozy-monitoring services that do not exist in the tenant cluster, or labels
+# every series tenant="cozy-monitoring".
+templates:
+  - templates/vmagent.yaml
+release:
+  name: cozy-monitoring-agents
+  namespace: cozy-monitoring
+tests:
+  - it: root install writes to both stores of global.target and labels series with it
+    set:
+      global.target: tenant-root
+    asserts:
+      - equal:
+          path: spec.remoteWrite
+          value:
+            - url: http://vminsert-shortterm.tenant-root.svc:8480/insert/0/prometheus
+            - url: http://vminsert-longterm.tenant-root.svc:8480/insert/0/prometheus
+      - equal:
+          path: spec.externalLabels.tenant
+          value: tenant-root
+
+  - it: tenant cluster writes only to the URLs its parent app passes
+    set:
+      vmagent:
+        externalLabels:
+          cluster: kubernetes-clusterprod
+          tenant: tenant-a
+        remoteWrite:
+          urls:
+            - http://vminsert-shortterm.tenant-a.svc.cozy.local:8480/insert/0/prometheus
+    asserts:
+      - equal:
+          path: spec.remoteWrite
+          value:
+            - url: http://vminsert-shortterm.tenant-a.svc.cozy.local:8480/insert/0/prometheus
+
+  - it: tenant cluster labels series with the owning tenant, not global.target
+    set:
+      vmagent:
+        externalLabels:
+          cluster: kubernetes-clusterprod
+          tenant: tenant-a
+    asserts:
+      - equal:
+          path: spec.externalLabels
+          value:
+            cluster: kubernetes-clusterprod
+            tenant: tenant-a
+
+  - it: tenant label stays a string when the value reads as a YAML scalar
+    set:
+      vmagent:
+        externalLabels:
+          tenant: "true"
+    asserts:
+      - equal:
+          path: spec.externalLabels.tenant
+          value: "true"