From 9d4221b890688060a5ce8a2be5ec9abd5e7d8475 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 12 Oct 2026 19:45:24 +0300
Subject: [PATCH] fix(monitoring): send alerts to every Alertmanager replica

vmalert had a single notifier, the headless vmalertmanager-alertmanager
Service. It resolves to all three pods, but each alert reached one replica
at a time and moved to another when vmalert reconnected. On clusters running
1.6.4 all active alerts sat on one replica while the other two had none, and
on one of them the alert count of each replica changed about 170 times a
day. The Alertmanager cluster deduplicates notifications only when every
replica receives every alert. The replica an alert left let it expire and
sent a resolved notification, and the replica it reached sent a new firing
one.

vmalert now lists each Alertmanager pod as its own notifier, using the
<name>-<i>.<service>.<namespace>.svc address the operator itself gives
StatefulSet pods. The replica count comes from one helper that also sets
spec.replicaCount of the VMAlertmanager, so the two cannot drift.

Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
 .../system/monitoring/templates/_helpers.tpl  |  9 +++
 .../monitoring/templates/alerta/alerta.yaml   |  2 +-
 .../monitoring/templates/vm/vmalert.yaml      |  4 +-
 .../tests/vmalert_notifiers_test.yaml         | 55 +++++++++++++++++++
 4 files changed, 68 insertions(+), 2 deletions(-)
 create mode 100644 packages/system/monitoring/tests/vmalert_notifiers_test.yaml

diff --git a/packages/system/monitoring/templates/_helpers.tpl b/packages/system/monitoring/templates/_helpers.tpl
index 59d8ba714..389720959 100644
--- a/packages/system/monitoring/templates/_helpers.tpl
+++ b/packages/system/monitoring/templates/_helpers.tpl
@@ -327,3 +327,12 @@
 {{- define "monitoring.tracingCentral" -}}
 {{- if and (ne .Release.Namespace "tenant-root") (eq (toString (dig "tracingCentral" false (.Values._namespace | default dict))) "true") -}}true{{- end -}}
 {{- end -}}
+
+{{- /*
+  Alertmanager replica count. vmalert lists each replica as its own
+  notifier from the same number, because the Alertmanager cluster only
+  deduplicates notifications when every replica receives every alert.
+*/}}
+{{- define "monitoring.alertmanager.replicas" -}}
+3
+{{- end -}}
diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml
index c6becd1bd..6376eaa9e 100644
--- a/packages/system/monitoring/templates/alerta/alerta.yaml
+++ b/packages/system/monitoring/templates/alerta/alerta.yaml
@@ -419,7 +419,7 @@ spec:
       apps.cozystack.io/application.group: apps.cozystack.io
       apps.cozystack.io/application.kind: Monitoring
       apps.cozystack.io/application.name: {{ $.Release.Name }}
-  replicaCount: 3
+  replicaCount: {{ include "monitoring.alertmanager.replicas" $ }}
   configSecret: alertmanager
   {{- if and $emailEnabled $email.authUsername }}
   secrets:
diff --git a/packages/system/monitoring/templates/vm/vmalert.yaml b/packages/system/monitoring/templates/vm/vmalert.yaml
index 8da03e152..b4f0c8aa1 100644
--- a/packages/system/monitoring/templates/vm/vmalert.yaml
+++ b/packages/system/monitoring/templates/vm/vmalert.yaml
@@ -16,7 +16,9 @@ spec:
   extraArgs:
     remoteWrite.disablePathAppend: "true"
   notifiers:
-  - url: http://vmalertmanager-alertmanager.{{ $.Release.Namespace }}.svc:9093
+  {{- range $i := until (int (include "monitoring.alertmanager.replicas" $)) }}
+  - url: http://vmalertmanager-alertmanager-{{ $i }}.vmalertmanager-alertmanager.{{ $.Release.Namespace }}.svc:9093
+  {{- end }}
   remoteRead:
     url: http://vmselect-{{ .name }}.{{ $.Release.Namespace }}.svc:8481/select/0/prometheus
   remoteWrite:
diff --git a/packages/system/monitoring/tests/vmalert_notifiers_test.yaml b/packages/system/monitoring/tests/vmalert_notifiers_test.yaml
new file mode 100644
index 000000000..6133b1f55
--- /dev/null
+++ b/packages/system/monitoring/tests/vmalert_notifiers_test.yaml
@@ -0,0 +1,55 @@
+suite: vmalert notifies every Alertmanager replica
+templates:
+  - templates/vm/vmalert.yaml
+  - templates/alerta/alerta.yaml
+
+release:
+  name: monitoring
+  namespace: tenant-root
+
+set:
+  _cluster:
+    root-host: example.org
+  _namespace:
+    host: tenant-root.example.org
+    ingress: tenant-root
+  metricsStorages:
+    - name: shortterm
+      retentionPeriod: "3d"
+      deduplicationInterval: "15s"
+
+# The Alertmanager cluster deduplicates notifications only when every replica
+# receives every alert. A notifier pointing at the headless Service reaches
+# one pod per connection, so an alert lives on whichever replica that
+# connection landed on and moves when vmalert reconnects. Each replica it
+# leaves sends a resolved notification and each one it reaches sends a new
+# firing one.
+tests:
+  - it: lists each Alertmanager pod as a notifier instead of the shared Service
+    template: templates/vm/vmalert.yaml
+    asserts:
+      - equal:
+          path: spec.notifiers
+          value:
+            - url: http://vmalertmanager-alertmanager-0.vmalertmanager-alertmanager.tenant-root.svc:9093
+            - url: http://vmalertmanager-alertmanager-1.vmalertmanager-alertmanager.tenant-root.svc:9093
+            - url: http://vmalertmanager-alertmanager-2.vmalertmanager-alertmanager.tenant-root.svc:9093
+
+  - it: runs as many Alertmanager replicas as vmalert has notifiers
+    template: templates/alerta/alerta.yaml
+    documentSelector:
+      path: kind
+      value: VMAlertmanager
+    asserts:
+      - equal:
+          path: spec.replicaCount
+          value: 3
+
+  - it: addresses the Alertmanager pods in the release namespace
+    template: templates/vm/vmalert.yaml
+    release:
+      namespace: tenant-foo
+    asserts:
+      - equal:
+          path: spec.notifiers[2].url
+          value: http://vmalertmanager-alertmanager-2.vmalertmanager-alertmanager.tenant-foo.svc:9093