fix(monitoring): send alerts to every Alertmanager replica

vmalert had a single notifier, the headless vmalertmanager-alertmanager
Service. It resolves to all three pods, but each alert reached one replica
at a time and moved to another when vmalert reconnected. On clusters running
1.6.4 all active alerts sat on one replica while the other two had none, and
on one of them the alert count of each replica changed about 170 times a
day. The Alertmanager cluster deduplicates notifications only when every
replica receives every alert. The replica an alert left let it expire and
sent a resolved notification, and the replica it reached sent a new firing
one.

vmalert now lists each Alertmanager pod as its own notifier, using the
<name>-<i>.<service>.<namespace>.svc address the operator itself gives
StatefulSet pods. The replica count comes from one helper that also sets
spec.replicaCount of the VMAlertmanager, so the two cannot drift.

Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>

packages/system/monitoring/templates/_helpers.tpl

@@ -327,3 +327,12 @@
 {{- define "monitoring.tracingCentral" -}}
 {{- if and (ne .Release.Namespace "tenant-root") (eq (toString (dig "tracingCentral" false (.Values._namespace | default dict))) "true") -}}true{{- end -}}
 {{- end -}}
+
+{{- /*
+  Alertmanager replica count. vmalert lists each replica as its own
+  notifier from the same number, because the Alertmanager cluster only
+  deduplicates notifications when every replica receives every alert.
+*/}}
+{{- define "monitoring.alertmanager.replicas" -}}
+3
+{{- end -}}

packages/system/monitoring/templates/alerta/alerta.yaml

@@ -419,7 +419,7 @@ spec:
       apps.cozystack.io/application.group: apps.cozystack.io
       apps.cozystack.io/application.kind: Monitoring
       apps.cozystack.io/application.name: {{ $.Release.Name }}
-  replicaCount: 3
+  replicaCount: {{ include "monitoring.alertmanager.replicas" $ }}
   configSecret: alertmanager
   {{- if and $emailEnabled $email.authUsername }}
   secrets:

packages/system/monitoring/templates/vm/vmalert.yaml

@@ -16,7 +16,9 @@ spec:
   extraArgs:
     remoteWrite.disablePathAppend: "true"
   notifiers:
-  - url: http://vmalertmanager-alertmanager.{{ $.Release.Namespace }}.svc:9093
+  {{- range $i := until (int (include "monitoring.alertmanager.replicas" $)) }}
+  - url: http://vmalertmanager-alertmanager-{{ $i }}.vmalertmanager-alertmanager.{{ $.Release.Namespace }}.svc:9093
+  {{- end }}
   remoteRead:
     url: http://vmselect-{{ .name }}.{{ $.Release.Namespace }}.svc:8481/select/0/prometheus
   remoteWrite:

packages/system/monitoring/tests/vmalert_notifiers_test.yaml

@@ -0,0 +1,55 @@
+suite: vmalert notifies every Alertmanager replica
+templates:
+  - templates/vm/vmalert.yaml
+  - templates/alerta/alerta.yaml
+
+release:
+  name: monitoring
+  namespace: tenant-root
+
+set:
+  _cluster:
+    root-host: example.org
+  _namespace:
+    host: tenant-root.example.org
+    ingress: tenant-root
+  metricsStorages:
+    - name: shortterm
+      retentionPeriod: "3d"
+      deduplicationInterval: "15s"
+
+# The Alertmanager cluster deduplicates notifications only when every replica
+# receives every alert. A notifier pointing at the headless Service reaches
+# one pod per connection, so an alert lives on whichever replica that
+# connection landed on and moves when vmalert reconnects. Each replica it
+# leaves sends a resolved notification and each one it reaches sends a new
+# firing one.
+tests:
+  - it: lists each Alertmanager pod as a notifier instead of the shared Service
+    template: templates/vm/vmalert.yaml
+    asserts:
+      - equal:
+          path: spec.notifiers
+          value:
+            - url: http://vmalertmanager-alertmanager-0.vmalertmanager-alertmanager.tenant-root.svc:9093
+            - url: http://vmalertmanager-alertmanager-1.vmalertmanager-alertmanager.tenant-root.svc:9093
+            - url: http://vmalertmanager-alertmanager-2.vmalertmanager-alertmanager.tenant-root.svc:9093
+
+  - it: runs as many Alertmanager replicas as vmalert has notifiers
+    template: templates/alerta/alerta.yaml
+    documentSelector:
+      path: kind
+      value: VMAlertmanager
+    asserts:
+      - equal:
+          path: spec.replicaCount
+          value: 3
+
+  - it: addresses the Alertmanager pods in the release namespace
+    template: templates/vm/vmalert.yaml
+    release:
+      namespace: tenant-foo
+    asserts:
+      - equal:
+          path: spec.notifiers[2].url
+          value: http://vmalertmanager-alertmanager-2.vmalertmanager-alertmanager.tenant-foo.svc:9093