From 7f7496c8d3f639993ad2a02f5ea2b1c7d9180265 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:16:36 +0700 Subject: [PATCH] fix(monitoring): remove the identity placeholder from scrape-health series vmagent appends up and scrape_* to each scrape after metric relabeling: in VictoriaMetrics v1.136.0, the version operator v0.68.4 deploys, lib/promscrape/scrapework.go addAutoMetrics passes needRelabel=false to writeRequestCtx.addRows, so addRow attaches the target labels but skips MetricRelabelConfigs. The "-" placeholder that the scrape class sets on targets without lineage was removed only in metricRelabelConfigs, so it stayed on the scrape-health series of every such target, including nodes and kube-state-metrics. The removal now runs in the VMAgent's inlineRelabelConfig, which the operator passes as -remoteWrite.relabelConfig. app/vmagent/remotewrite TryPush applies it to every series before remote write, scraped and generated alike, so targets without lineage keep the label set they had before the class. The exported_* drop stays in the class: only scraped samples can carry it. Assisted-by: LLM Signed-off-by: ZverGuy --- .../monitoring-agents/templates/vmagent.yaml | 18 +++++++++++------- .../vmagent_application_identity_test.yaml | 12 +++++++++++- .../monitoring/templates/vm/vmagent.yaml | 18 +++++++++++------- .../vmagent_application_identity_test.yaml | 12 +++++++++++- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index 29349f17e..ad26881b7 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -26,7 +26,8 @@ spec: # copy then collides, is renamed to exported_* and dropped. Under # honorLabels true the exporter's copy wins, so such a scrape must drop # these labels itself. A scrape object's own relabel rules run after these - # and must leave the labels alone. The placeholder is removed before storage. + # and must leave the labels alone. The placeholder is removed by the + # remote-write relabeling below. scrapeClasses: - name: application-identity default: true @@ -41,12 +42,15 @@ spec: metricRelabelConfigs: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) - {{- range list "group" "kind" "name" }} - - sourceLabels: [apps_cozystack_io_application_{{ . }}] - regex: "-" - targetLabel: apps_cozystack_io_application_{{ . }} - replacement: "" - {{- end }} + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} additionalScrapeConfigs: name: additional-scrape-configs key: prometheus-additional.yaml diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml index b59f9e3ee..95f330d50 100644 --- a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -46,13 +46,23 @@ tests: regex: (.+) targetLabel: apps_cozystack_io_application_name - - it: an exporter-supplied identity is dropped and the placeholder never reaches storage + - it: an exporter-supplied identity is dropped asserts: - equal: path: spec.scrapeClasses[0].metricRelabelConfigs value: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: - sourceLabels: [apps_cozystack_io_application_group] regex: "-" targetLabel: apps_cozystack_io_application_group diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index fa4c4fc93..2748ac4bf 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -30,7 +30,8 @@ spec: # copy then collides, is renamed to exported_* and dropped. Under # honorLabels true the exporter's copy wins, so such a scrape must drop # these labels itself. A scrape object's own relabel rules run after these - # and must leave the labels alone. The placeholder is removed before storage. + # and must leave the labels alone. The placeholder is removed by the + # remote-write relabeling below. scrapeClasses: - name: application-identity default: true @@ -45,12 +46,15 @@ spec: metricRelabelConfigs: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) - {{- range list "group" "kind" "name" }} - - sourceLabels: [apps_cozystack_io_application_{{ . }}] - regex: "-" - targetLabel: apps_cozystack_io_application_{{ . }} - replacement: "" - {{- end }} + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} {{- with .Values.vmagent.inlineScrapeConfig }} inlineScrapeConfig: | {{- . | nindent 4 }} diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml index da14330de..3a77fe6ee 100644 --- a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml +++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml @@ -42,13 +42,23 @@ tests: regex: (.+) targetLabel: apps_cozystack_io_application_name - - it: an exporter-supplied identity is dropped and the placeholder never reaches storage + - it: an exporter-supplied identity is dropped asserts: - equal: path: spec.scrapeClasses[0].metricRelabelConfigs value: - action: labeldrop regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: - sourceLabels: [apps_cozystack_io_application_group] regex: "-" targetLabel: apps_cozystack_io_application_group