diff --git a/packages/system/monitoring/templates/_helpers.tpl b/packages/system/monitoring/templates/_helpers.tpl index 59d8ba714..389720959 100644 --- a/packages/system/monitoring/templates/_helpers.tpl +++ b/packages/system/monitoring/templates/_helpers.tpl @@ -327,3 +327,12 @@ {{- define "monitoring.tracingCentral" -}} {{- if and (ne .Release.Namespace "tenant-root") (eq (toString (dig "tracingCentral" false (.Values._namespace | default dict))) "true") -}}true{{- end -}} {{- end -}} + +{{- /* + Alertmanager replica count. vmalert lists each replica as its own + notifier from the same number, because the Alertmanager cluster only + deduplicates notifications when every replica receives every alert. +*/}} +{{- define "monitoring.alertmanager.replicas" -}} +3 +{{- end -}} diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml index c6becd1bd..6376eaa9e 100644 --- a/packages/system/monitoring/templates/alerta/alerta.yaml +++ b/packages/system/monitoring/templates/alerta/alerta.yaml @@ -419,7 +419,7 @@ spec: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.kind: Monitoring apps.cozystack.io/application.name: {{ $.Release.Name }} - replicaCount: 3 + replicaCount: {{ include "monitoring.alertmanager.replicas" $ }} configSecret: alertmanager {{- if and $emailEnabled $email.authUsername }} secrets: diff --git a/packages/system/monitoring/templates/vm/vmalert.yaml b/packages/system/monitoring/templates/vm/vmalert.yaml index 8da03e152..b4f0c8aa1 100644 --- a/packages/system/monitoring/templates/vm/vmalert.yaml +++ b/packages/system/monitoring/templates/vm/vmalert.yaml @@ -16,7 +16,9 @@ spec: extraArgs: remoteWrite.disablePathAppend: "true" notifiers: - - url: http://vmalertmanager-alertmanager.{{ $.Release.Namespace }}.svc:9093 + {{- range $i := until (int (include "monitoring.alertmanager.replicas" $)) }} + - url: http://vmalertmanager-alertmanager-{{ $i }}.vmalertmanager-alertmanager.{{ $.Release.Namespace }}.svc:9093 + {{- end }} remoteRead: url: http://vmselect-{{ .name }}.{{ $.Release.Namespace }}.svc:8481/select/0/prometheus remoteWrite: diff --git a/packages/system/monitoring/tests/vmalert_notifiers_test.yaml b/packages/system/monitoring/tests/vmalert_notifiers_test.yaml new file mode 100644 index 000000000..6133b1f55 --- /dev/null +++ b/packages/system/monitoring/tests/vmalert_notifiers_test.yaml @@ -0,0 +1,55 @@ +suite: vmalert notifies every Alertmanager replica +templates: + - templates/vm/vmalert.yaml + - templates/alerta/alerta.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + metricsStorages: + - name: shortterm + retentionPeriod: "3d" + deduplicationInterval: "15s" + +# The Alertmanager cluster deduplicates notifications only when every replica +# receives every alert. A notifier pointing at the headless Service reaches +# one pod per connection, so an alert lives on whichever replica that +# connection landed on and moves when vmalert reconnects. Each replica it +# leaves sends a resolved notification and each one it reaches sends a new +# firing one. +tests: + - it: lists each Alertmanager pod as a notifier instead of the shared Service + template: templates/vm/vmalert.yaml + asserts: + - equal: + path: spec.notifiers + value: + - url: http://vmalertmanager-alertmanager-0.vmalertmanager-alertmanager.tenant-root.svc:9093 + - url: http://vmalertmanager-alertmanager-1.vmalertmanager-alertmanager.tenant-root.svc:9093 + - url: http://vmalertmanager-alertmanager-2.vmalertmanager-alertmanager.tenant-root.svc:9093 + + - it: runs as many Alertmanager replicas as vmalert has notifiers + template: templates/alerta/alerta.yaml + documentSelector: + path: kind + value: VMAlertmanager + asserts: + - equal: + path: spec.replicaCount + value: 3 + + - it: addresses the Alertmanager pods in the release namespace + template: templates/vm/vmalert.yaml + release: + namespace: tenant-foo + asserts: + - equal: + path: spec.notifiers[2].url + value: http://vmalertmanager-alertmanager-2.vmalertmanager-alertmanager.tenant-foo.svc:9093