From f98ee76d2db3ab3acea796ce981374461e82caaa Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 04:16:56 +0700 Subject: [PATCH 01/10] feat(monitoring): stamp application identity on cluster-wide scrapes Scoping a metrics query to one application needs the application's identity on the series themselves, and it is only a boundary if the scraped pod cannot choose it. A default scrape class on the cluster-wide vmagent copies the pod's lineage labels into apps_cozystack_io_application_{group,kind,name} at scrape time. The target always carries all three labels, with a placeholder when the pod has no lineage, so an exporter's own copy always collides, is renamed to exported_* under honorLabels false, and is dropped. Without the placeholder a pod lacking lineage could publish any identity unchallenged. The placeholder is removed again before storage, so series of pods without lineage look as they did before. Scrapes with honorLabels true keep the exporter's value; none of them scrapes a tenant-controlled exporter. Assisted-by: LLM Signed-off-by: ZverGuy --- .../monitoring-agents/templates/vmagent.yaml | 25 +++++++ .../vmagent_application_identity_test.yaml | 67 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index 00ab63c10..f0b66bd59 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -20,6 +20,31 @@ spec: scrapeInterval: 30s selectAllByDefault: true + # The target always carries all three identity labels, "-" when the pod + # has no lineage ("-" is not a valid Kubernetes label value, so it never + # collides with a real one). That makes every exporter-supplied copy + # collide and be renamed to exported_* (honorLabels false), where it is + # dropped; the placeholder itself is removed before storage. + scrapeClasses: + - name: application-identity + default: true + relabelConfigs: + {{- range list "group" "kind" "name" }} + - targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}] + regex: (.+) + targetLabel: apps_cozystack_io_application_{{ . }} + {{- end }} + metricRelabelConfigs: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} additionalScrapeConfigs: name: additional-scrape-configs key: prometheus-additional.yaml diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml new file mode 100644 index 000000000..601f1c926 --- /dev/null +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -0,0 +1,67 @@ +suite: cluster-wide vmagent stamps application identity on exporter series +templates: + - templates/vmagent.yaml +release: + name: monitoring-agents + namespace: cozy-monitoring + +# The tenant-resource-metrics proxy scopes a query to one application by +# apps_cozystack_io_application_{group,kind,name}. The labels are only a +# boundary if the scraped pod cannot choose them: the target always carries +# all three (a placeholder when the pod has no lineage), so with +# honorLabels false an exporter's own copy is renamed to exported_* and +# dropped. The same rules live in the tenant vmagent of the monitoring +# package and are pinned there by an identical suite. +tests: + - it: every scrape gets the application identity class unless it names another + asserts: + - equal: + path: spec.scrapeClasses[0].name + value: application-identity + - equal: + path: spec.scrapeClasses[0].default + value: true + - lengthEqual: + path: spec.scrapeClasses + count: 1 + + - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none + asserts: + - equal: + path: spec.scrapeClasses[0].relabelConfigs + value: + - targetLabel: apps_cozystack_io_application_group + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group] + regex: (.+) + targetLabel: apps_cozystack_io_application_group + - targetLabel: apps_cozystack_io_application_kind + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind] + regex: (.+) + targetLabel: apps_cozystack_io_application_kind + - targetLabel: apps_cozystack_io_application_name + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name] + regex: (.+) + targetLabel: apps_cozystack_io_application_name + + - it: an exporter-supplied identity is dropped and the placeholder never reaches storage + asserts: + - equal: + path: spec.scrapeClasses[0].metricRelabelConfigs + value: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + - sourceLabels: [apps_cozystack_io_application_group] + regex: "-" + targetLabel: apps_cozystack_io_application_group + replacement: "" + - sourceLabels: [apps_cozystack_io_application_kind] + regex: "-" + targetLabel: apps_cozystack_io_application_kind + replacement: "" + - sourceLabels: [apps_cozystack_io_application_name] + regex: "-" + targetLabel: apps_cozystack_io_application_name + replacement: "" From 14f815a5cc543ba35eeec1d407cc07a3b5b7fea1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 04:16:56 +0700 Subject: [PATCH 02/10] feat(monitoring): stamp application identity on tenant vmagent scrapes Application pods in a tenant's namespaces are also scraped by the tenant's own vmagent, and its series need the same identity labels and the same protection against an exporter choosing them, or the two stores would disagree about which application a series belongs to. The class is identical to the cluster-wide one in monitoring-agents. Assisted-by: LLM Signed-off-by: ZverGuy --- .../monitoring/templates/vm/vmagent.yaml | 25 ++++++++ .../vmagent_application_identity_test.yaml | 62 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 packages/system/monitoring/tests/vmagent_application_identity_test.yaml diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index 98bff1e14..f65cd287e 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -24,6 +24,31 @@ spec: scrapeInterval: 30s selectAllByDefault: false + # The target always carries all three identity labels, "-" when the pod + # has no lineage ("-" is not a valid Kubernetes label value, so it never + # collides with a real one). That makes every exporter-supplied copy + # collide and be renamed to exported_* (honorLabels false), where it is + # dropped; the placeholder itself is removed before storage. + scrapeClasses: + - name: application-identity + default: true + relabelConfigs: + {{- range list "group" "kind" "name" }} + - targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}] + regex: (.+) + targetLabel: apps_cozystack_io_application_{{ . }} + {{- end }} + metricRelabelConfigs: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} {{- with .Values.vmagent.inlineScrapeConfig }} inlineScrapeConfig: | {{- . | nindent 4 }} diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml new file mode 100644 index 000000000..f36105f83 --- /dev/null +++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml @@ -0,0 +1,62 @@ +suite: tenant vmagent stamps application identity on exporter series +templates: + - templates/vm/vmagent.yaml +release: + name: monitoring + namespace: tenant-foo + +# Same rules as the cluster-wide vmagent in monitoring-agents, pinned there +# by an identical suite; see it for why the placeholder and the drop exist. +tests: + - it: every scrape gets the application identity class unless it names another + asserts: + - equal: + path: spec.scrapeClasses[0].name + value: application-identity + - equal: + path: spec.scrapeClasses[0].default + value: true + - lengthEqual: + path: spec.scrapeClasses + count: 1 + + - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none + asserts: + - equal: + path: spec.scrapeClasses[0].relabelConfigs + value: + - targetLabel: apps_cozystack_io_application_group + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group] + regex: (.+) + targetLabel: apps_cozystack_io_application_group + - targetLabel: apps_cozystack_io_application_kind + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind] + regex: (.+) + targetLabel: apps_cozystack_io_application_kind + - targetLabel: apps_cozystack_io_application_name + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name] + regex: (.+) + targetLabel: apps_cozystack_io_application_name + + - it: an exporter-supplied identity is dropped and the placeholder never reaches storage + asserts: + - equal: + path: spec.scrapeClasses[0].metricRelabelConfigs + value: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + - sourceLabels: [apps_cozystack_io_application_group] + regex: "-" + targetLabel: apps_cozystack_io_application_group + replacement: "" + - sourceLabels: [apps_cozystack_io_application_kind] + regex: "-" + targetLabel: apps_cozystack_io_application_kind + replacement: "" + - sourceLabels: [apps_cozystack_io_application_name] + regex: "-" + targetLabel: apps_cozystack_io_application_name + replacement: "" From 0e5557d0baef802125b88d438d741055da5ee408 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 04:26:04 +0700 Subject: [PATCH 03/10] fix(ingress): keep application identity off ingress-nginx series The vmagent scrape class stamps the scraped pod's application identity on its series. ingress-nginx series describe the Ingress objects the controller serves, in whatever namespace they live, so the controller's own identity would attribute another tenant's traffic to the Ingress application. And because these scrapes honor exporter labels, an exporter-supplied identity would win over the class. Dropping the labels leaves these series to the pod and namespace scopes. Assisted-by: LLM Signed-off-by: ZverGuy --- .../extra/ingress/templates/nginx-scrape.yaml | 10 +++++++++ .../tests/application_identity_test.yaml | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 packages/extra/ingress/tests/application_identity_test.yaml diff --git a/packages/extra/ingress/templates/nginx-scrape.yaml b/packages/extra/ingress/templates/nginx-scrape.yaml index d04c5b651..780f5e3a9 100644 --- a/packages/extra/ingress/templates/nginx-scrape.yaml +++ b/packages/extra/ingress/templates/nginx-scrape.yaml @@ -9,6 +9,10 @@ spec: matchNames: - {{ .Release.Namespace }} podMetricsEndpoints: + # The series describe the Ingress objects served, in any namespace, not + # this controller, so the application identity the vmagent scrape class + # stamps from the controller pod would misattribute them; and with + # honorLabels an exporter-supplied identity would win over the class. - port: metrics honorLabels: true interval: 30s @@ -19,6 +23,9 @@ spec: targetLabel: node - targetLabel: tier replacement: cluster + metricRelabelConfigs: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) selector: matchLabels: app.kubernetes.io/component: controller @@ -46,6 +53,9 @@ spec: targetLabel: controller - targetLabel: tier replacement: cluster + metricRelabelConfigs: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) selector: matchLabels: app.kubernetes.io/component: controller diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml new file mode 100644 index 000000000..e1be5ca4f --- /dev/null +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -0,0 +1,21 @@ +suite: ingress-nginx series do not carry an application identity +templates: + - templates/nginx-scrape.yaml +release: + name: ingress + namespace: tenant-root + +# The vmagent scrape class stamps the scraped pod's application identity on +# every series. ingress-nginx series describe the Ingress objects it serves, +# in whatever namespace they live (honorLabels keeps their namespace), so +# the controller's identity would misattribute them, and with honorLabels +# true an exporter-supplied identity would win over the class. Both +# endpoints drop the identity instead. +tests: + - it: drops the identity on every endpoint + asserts: + - equal: + path: spec.podMetricsEndpoints[0].metricRelabelConfigs + value: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) From dca95eb753fa31271eb7b3cbbe3a0312da138846 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 04:26:04 +0700 Subject: [PATCH 04/10] docs(monitoring): state when the identity scrape class can be overridden The exported_* rename only protects scrapes that do not honor exporter labels, and a scrape object's own relabel rules run after the class, so both can undo it. Saying so next to the rules keeps the next scrape author from assuming the protection is unconditional. Assisted-by: LLM Signed-off-by: ZverGuy --- packages/system/monitoring-agents/templates/vmagent.yaml | 8 +++++--- packages/system/monitoring/templates/vm/vmagent.yaml | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index f0b66bd59..29349f17e 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -22,9 +22,11 @@ spec: selectAllByDefault: true # The target always carries all three identity labels, "-" when the pod # has no lineage ("-" is not a valid Kubernetes label value, so it never - # collides with a real one). That makes every exporter-supplied copy - # collide and be renamed to exported_* (honorLabels false), where it is - # dropped; the placeholder itself is removed before storage. + # collides with a real one). Under honorLabels false every exporter-supplied + # copy then collides, is renamed to exported_* and dropped. Under + # honorLabels true the exporter's copy wins, so such a scrape must drop + # these labels itself. A scrape object's own relabel rules run after these + # and must leave the labels alone. The placeholder is removed before storage. scrapeClasses: - name: application-identity default: true diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index f65cd287e..fa4c4fc93 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -26,9 +26,11 @@ spec: selectAllByDefault: false # The target always carries all three identity labels, "-" when the pod # has no lineage ("-" is not a valid Kubernetes label value, so it never - # collides with a real one). That makes every exporter-supplied copy - # collide and be renamed to exported_* (honorLabels false), where it is - # dropped; the placeholder itself is removed before storage. + # collides with a real one). Under honorLabels false every exporter-supplied + # copy then collides, is renamed to exported_* and dropped. Under + # honorLabels true the exporter's copy wins, so such a scrape must drop + # these labels itself. A scrape object's own relabel rules run after these + # and must leave the labels alone. The placeholder is removed before storage. scrapeClasses: - name: application-identity default: true From 5a6086c7a1756499f10fd869e20e7bd3c14a5e17 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 04:30:32 +0700 Subject: [PATCH 05/10] test(monitoring): pin the endpoint count the ingress identity test relies on The ingress assertion reads only the first endpoint of each scrape, so a second endpoint added without the label drop would go unnoticed. Pinning the count makes such an addition fail the suite. Assisted-by: LLM Signed-off-by: ZverGuy --- packages/extra/ingress/tests/application_identity_test.yaml | 3 +++ .../tests/vmagent_application_identity_test.yaml | 2 +- .../monitoring/tests/vmagent_application_identity_test.yaml | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml index e1be5ca4f..0e53a2756 100644 --- a/packages/extra/ingress/tests/application_identity_test.yaml +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -14,6 +14,9 @@ release: tests: - it: drops the identity on every endpoint asserts: + - lengthEqual: + path: spec.podMetricsEndpoints + count: 1 - equal: path: spec.podMetricsEndpoints[0].metricRelabelConfigs value: diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml index 601f1c926..b59f9e3ee 100644 --- a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -11,7 +11,7 @@ release: # all three (a placeholder when the pod has no lineage), so with # honorLabels false an exporter's own copy is renamed to exported_* and # dropped. The same rules live in the tenant vmagent of the monitoring -# package and are pinned there by an identical suite. +# package and are pinned there by the same assertions. tests: - it: every scrape gets the application identity class unless it names another asserts: diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml index f36105f83..da14330de 100644 --- a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml @@ -6,7 +6,8 @@ release: namespace: tenant-foo # Same rules as the cluster-wide vmagent in monitoring-agents, pinned there -# by an identical suite; see it for why the placeholder and the drop exist. +# by the same assertions; see that suite for why the placeholder and the +# drop exist. tests: - it: every scrape gets the application identity class unless it names another asserts: From 7f7496c8d3f639993ad2a02f5ea2b1c7d9180265 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:16:36 +0700 Subject: [PATCH 06/10] fix(monitoring): remove the identity placeholder from scrape-health series vmagent appends up and scrape_* to each scrape after metric relabeling: in VictoriaMetrics v1.136.0, the version operator v0.68.4 deploys, lib/promscrape/scrapework.go addAutoMetrics passes needRelabel=false to writeRequestCtx.addRows, so addRow attaches the target labels but skips MetricRelabelConfigs. The "-" placeholder that the scrape class sets on targets without lineage was removed only in metricRelabelConfigs, so it stayed on the scrape-health series of every such target, including nodes and kube-state-metrics. The removal now runs in the VMAgent's inlineRelabelConfig, which the operator passes as -remoteWrite.relabelConfig. app/vmagent/remotewrite TryPush applies it to every series before remote write, scraped and generated alike, so targets without lineage keep the label set they had before the class. The exported_* drop stays in the class: only scraped samples can carry it. Assisted-by: LLM Signed-off-by: ZverGuy --- .../monitoring-agents/templates/vmagent.yaml | 18 +++++++++++------- .../vmagent_application_identity_test.yaml | 12 +++++++++++- .../monitoring/templates/vm/vmagent.yaml | 18 +++++++++++------- .../vmagent_application_identity_test.yaml | 12 +++++++++++- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index 29349f17e..ad26881b7 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -26,7 +26,8 @@ spec: # copy then collides, is renamed to exported_* and dropped. Under # honorLabels true the exporter's copy wins, so such a scrape must drop # these labels itself. A scrape object's own relabel rules run after these - # and must leave the labels alone. The placeholder is removed before storage. + # and must leave the labels alone. The placeholder is removed by the + # remote-write relabeling below. scrapeClasses: - name: application-identity default: true @@ -41,12 +42,15 @@ spec: metricRelabelConfigs: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) - {{- range list "group" "kind" "name" }} - - sourceLabels: [apps_cozystack_io_application_{{ . }}] - regex: "-" - targetLabel: apps_cozystack_io_application_{{ . }} - replacement: "" - {{- end }} + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} additionalScrapeConfigs: name: additional-scrape-configs key: prometheus-additional.yaml diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml index b59f9e3ee..95f330d50 100644 --- a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -46,13 +46,23 @@ tests: regex: (.+) targetLabel: apps_cozystack_io_application_name - - it: an exporter-supplied identity is dropped and the placeholder never reaches storage + - it: an exporter-supplied identity is dropped asserts: - equal: path: spec.scrapeClasses[0].metricRelabelConfigs value: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: - sourceLabels: [apps_cozystack_io_application_group] regex: "-" targetLabel: apps_cozystack_io_application_group diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index fa4c4fc93..2748ac4bf 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -30,7 +30,8 @@ spec: # copy then collides, is renamed to exported_* and dropped. Under # honorLabels true the exporter's copy wins, so such a scrape must drop # these labels itself. A scrape object's own relabel rules run after these - # and must leave the labels alone. The placeholder is removed before storage. + # and must leave the labels alone. The placeholder is removed by the + # remote-write relabeling below. scrapeClasses: - name: application-identity default: true @@ -45,12 +46,15 @@ spec: metricRelabelConfigs: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) - {{- range list "group" "kind" "name" }} - - sourceLabels: [apps_cozystack_io_application_{{ . }}] - regex: "-" - targetLabel: apps_cozystack_io_application_{{ . }} - replacement: "" - {{- end }} + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} {{- with .Values.vmagent.inlineScrapeConfig }} inlineScrapeConfig: | {{- . | nindent 4 }} diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml index da14330de..3a77fe6ee 100644 --- a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml @@ -42,13 +42,23 @@ tests: regex: (.+) targetLabel: apps_cozystack_io_application_name - - it: an exporter-supplied identity is dropped and the placeholder never reaches storage + - it: an exporter-supplied identity is dropped asserts: - equal: path: spec.scrapeClasses[0].metricRelabelConfigs value: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: - sourceLabels: [apps_cozystack_io_application_group] regex: "-" targetLabel: apps_cozystack_io_application_group From dab6035a97f9e5d960294bc7c24368b75df7f026 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:17:46 +0700 Subject: [PATCH 07/10] fix(ingress): keep application identity off ingress-nginx scrape-health series Dropping the identity in metricRelabelConfigs reaches only the scraped samples. vmagent adds up and scrape_* with the target labels after metric relabeling (VictoriaMetrics v1.136.0 lib/promscrape addAutoMetrics, addRows with needRelabel=false), so those series still carried the controller pod's identity. Dropping it from the target as well leaves every ingress-nginx series with the label set it had before the scrape class. The metric-stage drop stays: with honorLabels true an exporter-supplied identity is not renamed and would otherwise survive. Assisted-by: LLM Signed-off-by: ZverGuy --- packages/extra/ingress/templates/nginx-scrape.yaml | 7 ++++++- .../extra/ingress/tests/application_identity_test.yaml | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/extra/ingress/templates/nginx-scrape.yaml b/packages/extra/ingress/templates/nginx-scrape.yaml index 780f5e3a9..b98f18fe8 100644 --- a/packages/extra/ingress/templates/nginx-scrape.yaml +++ b/packages/extra/ingress/templates/nginx-scrape.yaml @@ -12,7 +12,8 @@ spec: # The series describe the Ingress objects served, in any namespace, not # this controller, so the application identity the vmagent scrape class # stamps from the controller pod would misattribute them; and with - # honorLabels an exporter-supplied identity would win over the class. + # honorLabels an exporter-supplied identity would win over the class. The + # target-stage drop also covers up and scrape_*, which skip metric relabeling. - port: metrics honorLabels: true interval: 30s @@ -23,6 +24,8 @@ spec: targetLabel: node - targetLabel: tier replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) metricRelabelConfigs: - action: labeldrop regex: apps_cozystack_io_application_(group|kind|name) @@ -53,6 +56,8 @@ spec: targetLabel: controller - targetLabel: tier replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) metricRelabelConfigs: - action: labeldrop regex: apps_cozystack_io_application_(group|kind|name) diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml index 0e53a2756..4fed325d2 100644 --- a/packages/extra/ingress/tests/application_identity_test.yaml +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -10,13 +10,20 @@ release: # in whatever namespace they live (honorLabels keeps their namespace), so # the controller's identity would misattribute them, and with honorLabels # true an exporter-supplied identity would win over the class. Both -# endpoints drop the identity instead. +# endpoints drop the identity instead: from the target, because vmagent +# adds up and scrape_* with the target labels after metric relabeling, and +# from the scraped samples, because an exporter's copy is not renamed. tests: - it: drops the identity on every endpoint asserts: - lengthEqual: path: spec.podMetricsEndpoints count: 1 + - contains: + path: spec.podMetricsEndpoints[0].relabelConfigs + content: + action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) - equal: path: spec.podMetricsEndpoints[0].metricRelabelConfigs value: From 3390100226a60f8e399fc589f42c5dbe32af4e27 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:21:26 +0700 Subject: [PATCH 08/10] docs(monitoring): state which scrapes the identity class covers The comment promised identity labels on every target and forbade scrape objects from touching them. Neither holds: operator v0.68.4 applies only a class's auth to VMProbe (factory/vmagent/probe.go), raw scrape configs bypass classes, and the cadvisor, kubelet and ingress-nginx scrapes drop the labels on purpose. The real invariant for a scrape object is that it never sets them; dropping them is safe. Assisted-by: LLM Signed-off-by: ZverGuy --- .../monitoring-agents/templates/vmagent.yaml | 18 ++++++++++-------- .../vmagent_application_identity_test.yaml | 4 ++-- .../monitoring/templates/vm/vmagent.yaml | 18 ++++++++++-------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index ad26881b7..e001fc40f 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -20,14 +20,16 @@ spec: scrapeInterval: 30s selectAllByDefault: true - # The target always carries all three identity labels, "-" when the pod - # has no lineage ("-" is not a valid Kubernetes label value, so it never - # collides with a real one). Under honorLabels false every exporter-supplied - # copy then collides, is renamed to exported_* and dropped. Under - # honorLabels true the exporter's copy wins, so such a scrape must drop - # these labels itself. A scrape object's own relabel rules run after these - # and must leave the labels alone. The placeholder is removed by the - # remote-write relabeling below. + # The class sets all three identity labels on every target it covers, "-" + # when the pod has no lineage ("-" is not a valid Kubernetes label value, + # so it never collides with a real one). Under honorLabels false every + # exporter-supplied copy then collides, is renamed to exported_* and + # dropped. Under honorLabels true the exporter's copy wins, so such a + # scrape must drop these labels itself. A scrape object's own relabel + # rules run after these: they may drop the labels but must never set them. + # Not covered: VMProbe (operator v0.68.4 takes only auth from a class) and + # raw scrape configs. The placeholder is removed by the remote-write + # relabeling below. scrapeClasses: - name: application-identity default: true diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml index 95f330d50..4e4b93385 100644 --- a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -7,8 +7,8 @@ release: # The tenant-resource-metrics proxy scopes a query to one application by # apps_cozystack_io_application_{group,kind,name}. The labels are only a -# boundary if the scraped pod cannot choose them: the target always carries -# all three (a placeholder when the pod has no lineage), so with +# boundary if the scraped pod cannot choose them: the class sets all three +# on every target it covers (a placeholder when the pod has no lineage), so with # honorLabels false an exporter's own copy is renamed to exported_* and # dropped. The same rules live in the tenant vmagent of the monitoring # package and are pinned there by the same assertions. diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index 2748ac4bf..919fdef63 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -24,14 +24,16 @@ spec: scrapeInterval: 30s selectAllByDefault: false - # The target always carries all three identity labels, "-" when the pod - # has no lineage ("-" is not a valid Kubernetes label value, so it never - # collides with a real one). Under honorLabels false every exporter-supplied - # copy then collides, is renamed to exported_* and dropped. Under - # honorLabels true the exporter's copy wins, so such a scrape must drop - # these labels itself. A scrape object's own relabel rules run after these - # and must leave the labels alone. The placeholder is removed by the - # remote-write relabeling below. + # The class sets all three identity labels on every target it covers, "-" + # when the pod has no lineage ("-" is not a valid Kubernetes label value, + # so it never collides with a real one). Under honorLabels false every + # exporter-supplied copy then collides, is renamed to exported_* and + # dropped. Under honorLabels true the exporter's copy wins, so such a + # scrape must drop these labels itself. A scrape object's own relabel + # rules run after these: they may drop the labels but must never set them. + # Not covered: VMProbe (operator v0.68.4 takes only auth from a class) and + # raw scrape configs. The placeholder is removed by the remote-write + # relabeling below. scrapeClasses: - name: application-identity default: true From 1018eaa3b632fbd708c57c6bda11c4d946c88d4a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:27:02 +0700 Subject: [PATCH 09/10] docs(ingress): name the controller series that lose application identity The comment said every ingress-nginx series describes served Ingress objects. The metrics port also serves the controller's own series (nginx process, reload status, go_*, up, scrape_*), and dropping the identity on the whole endpoint removes it from those as well. Say so, so the lost attribution reads as a decision rather than an oversight. Assisted-by: LLM Signed-off-by: ZverGuy --- packages/extra/ingress/templates/nginx-scrape.yaml | 12 +++++++----- .../ingress/tests/application_identity_test.yaml | 9 +++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/extra/ingress/templates/nginx-scrape.yaml b/packages/extra/ingress/templates/nginx-scrape.yaml index b98f18fe8..4917206de 100644 --- a/packages/extra/ingress/templates/nginx-scrape.yaml +++ b/packages/extra/ingress/templates/nginx-scrape.yaml @@ -9,11 +9,13 @@ spec: matchNames: - {{ .Release.Namespace }} podMetricsEndpoints: - # The series describe the Ingress objects served, in any namespace, not - # this controller, so the application identity the vmagent scrape class - # stamps from the controller pod would misattribute them; and with - # honorLabels an exporter-supplied identity would win over the class. The - # target-stage drop also covers up and scrape_*, which skip metric relabeling. + # Most series describe the Ingress objects served, in any namespace, so + # the application identity the vmagent scrape class stamps from the + # controller pod would misattribute them; and with honorLabels an + # exporter-supplied identity would win over the class. The controller's + # own series (nginx process, reload status, go_*, up, scrape_*) lose the + # Ingress application's identity too; that is the accepted cost. The + # target-stage drop covers up and scrape_*, which skip metric relabeling. - port: metrics honorLabels: true interval: 30s diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml index 4fed325d2..db781a102 100644 --- a/packages/extra/ingress/tests/application_identity_test.yaml +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -6,10 +6,11 @@ release: namespace: tenant-root # The vmagent scrape class stamps the scraped pod's application identity on -# every series. ingress-nginx series describe the Ingress objects it serves, -# in whatever namespace they live (honorLabels keeps their namespace), so -# the controller's identity would misattribute them, and with honorLabels -# true an exporter-supplied identity would win over the class. Both +# every series. Most ingress-nginx series describe the Ingress objects it +# serves, in whatever namespace they live (honorLabels keeps their +# namespace), so the controller's identity would misattribute them, and with +# honorLabels true an exporter-supplied identity would win over the class. +# The controller's own health series lose that identity too, by design. Both # endpoints drop the identity instead: from the target, because vmagent # adds up and scrape_* with the target labels after metric relabeling, and # from the scraped samples, because an exporter's copy is not renamed. From 699b5ce78e7d7c5d5b49d556eed086c6fe3d1a6d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:35:37 +0700 Subject: [PATCH 10/10] test(ingress): pin the position of the target-stage identity drop A containment check passes even if a rule added after the drop sets the identity labels again. Asserting both endpoints' full target relabel lists fixes the drop as the last rule, so such a change fails the test. Assisted-by: LLM Signed-off-by: ZverGuy --- .../tests/application_identity_test.yaml | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml index db781a102..4914049c8 100644 --- a/packages/extra/ingress/tests/application_identity_test.yaml +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -20,13 +20,43 @@ tests: - lengthEqual: path: spec.podMetricsEndpoints count: 1 - - contains: - path: spec.podMetricsEndpoints[0].relabelConfigs - content: - action: labeldrop - regex: apps_cozystack_io_application_(group|kind|name) - equal: path: spec.podMetricsEndpoints[0].metricRelabelConfigs value: - action: labeldrop regex: apps_cozystack_io_application_(group|kind|name) + + # Endpoint rules run after the scrape class rules, so the drop takes + # effect wherever it sits in this list; it is pinned last so that no + # later rule can set the identity again. + - it: drops the identity from the basic endpoint's target as its last rule + documentIndex: 0 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].relabelConfigs + value: + - replacement: nginx-ingress-controller + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - targetLabel: tier + replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + + - it: drops the identity from the detailed endpoint's target as its last rule + documentIndex: 1 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].relabelConfigs + value: + - replacement: nginx-ingress-controller + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + targetLabel: controller + - targetLabel: tier + replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name)