From 89028b455ef2fbf4772672c2fe6342150efc3200 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:16:45 +0300 Subject: [PATCH] fix(monitoring): route InfoInhibitor to the blackhole receiver InfoInhibitor exists to be the source side of an Alertmanager inhibit rule and, as its own description says, to go to a null receiver. The platform Alertmanager config has no inhibit rule, and the rule has carried severity major since it was imported alongside Alerta, so Alerta shows it one level below critical. Alerta's Telegram and Slack plugins forward it by default, and a notify_email label inherited from the info alert gets it emailed too. It fires whenever a rule labelled severity="info" is active with no warning or critical alert in the same namespace and cluster. The platform's own rules no longer use info, but rules it does not write still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt generates KubeVirtVMGuestMemoryAvailableLow, and operators add their own. Each of them turns into a major InfoInhibitor in Alerta. none is the severity upstream kube-prometheus gives this rule, and the platform config already sends severity="none" to the blackhole ahead of every route that delivers. The severity contract refuses none because such an alert is never delivered, so InfoInhibitor is named as the one alert for which that is the point. Tests hold both ends, so neither the label nor the route order can drift back to delivering it. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- hack/alert-severity-contract.bats | 89 +++++++++++++++++++ .../alerts/general.rules.yaml | 2 +- .../tests/alertmanager_blackhole_test.yaml | 45 ++++++++++ 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 packages/system/monitoring/tests/alertmanager_blackhole_test.yaml diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats index e1315eb96..44f197000 100644 --- a/hack/alert-severity-contract.bats +++ b/hack/alert-severity-contract.bats @@ -9,6 +9,10 @@ load test_helper # scope: they are re-vendored by `make update`, so a bad severity there is # fixed upstream or in patches/, not by editing the label. ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown " +# The alerts that exist only to be dropped, the one place `none` is right. +# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the +# platform config does not have, so delivering it at any severity is noise. +ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor " helm_actions_stripped() { awk ' @@ -104,6 +108,14 @@ alert_severity_contract() { break fi done + if [ "$severity" = "none" ]; then + for blackholed in $ALERT_SEVERITY_BLACKHOLED; do + if [ "$blackholed" = "$alert" ]; then + accepted=1 + break + fi + done + fi if [ "$accepted" -eq 0 ]; then echo "$file: $alert -> '$severity'" >&2 bad=$((bad + 1)) @@ -128,6 +140,83 @@ alert_severity_contract() { alert_severity_contract } +@test "InfoInhibitor is labelled for the blackhole route" { + rows=$(mktemp) || return 1 + if ! alert_severity_rows > "$rows"; then + rm -f "$rows" + return 1 + fi + tab=$(printf '\t') + found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true) + delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true) + rm -f "$rows" + [ "$found" -gt 0 ] + [ "$delivered" -eq 0 ] +} + +@test "severity none passes only on an alert that exists to be dropped" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + inhibitor_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$? + + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: silent' \ + ' rules:' \ + ' - alert: SilentAlert' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/01-silent.yaml" + git -C "$fixture" add packages/system/example/alerts/01-silent.yaml + + silent_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$? + rm -rf "$fixture" + [ "$inhibitor_succeeded" -eq 0 ] + [ "$silent_succeeded" -ne 0 ] +} + +@test "an alert that exists to be dropped still fails on a refused severity" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: info' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -ne 0 ] +} + @test "a parser failure cannot leave the contract green" { fixture=$(mktemp -d) mkdir -p "$fixture/packages/system/example/alerts" diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml index 924870c40..45dd6100f 100644 --- a/packages/system/monitoring-agents/alerts/general.rules.yaml +++ b/packages/system/monitoring-agents/alerts/general.rules.yaml @@ -52,6 +52,6 @@ spec: != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 labels: - severity: major + severity: none exported_instance: global service: general.rules diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml new file mode 100644 index 000000000..e987bd405 --- /dev/null +++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml @@ -0,0 +1,45 @@ +suite: Alertmanager blackhole route +templates: + - templates/alerta/alerta.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + +# Rules labelled severity="none", InfoInhibitor among them, exist to be +# dropped, and Alerta answers that severity with a 500. Only the heartbeat +# route may come ahead of the blackhole, so no delivering route claims them. +tests: + - it: drops severity none ahead of every delivering route + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + + - it: keeps severity none out of email even when its alert name is listed + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + alertnames: + - InfoInhibitor + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'