fix(monitoring): route InfoInhibitor to the blackhole receiver

InfoInhibitor exists to be the source side of an Alertmanager inhibit
rule and, as its own description says, to go to a null receiver. The
platform Alertmanager config has no inhibit rule, and the rule has
carried severity major since it was imported alongside Alerta, so
Alerta shows it one level below critical. Alerta's Telegram and Slack
plugins forward it by default, and a notify_email label inherited from
the info alert gets it emailed too.

It fires whenever a rule labelled severity="info" is active with no
warning or critical alert in the same namespace and cluster. The
platform's own rules no longer use info, but rules it does not write
still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt
generates KubeVirtVMGuestMemoryAvailableLow, and operators add their
own. Each of them turns into a major InfoInhibitor in Alerta.

none is the severity upstream kube-prometheus gives this rule, and the
platform config already sends severity="none" to the blackhole ahead
of every route that delivers. The severity contract refuses none
because such an alert is never delivered, so InfoInhibitor is named as
the one alert for which that is the point. Tests hold both ends, so
neither the label nor the route order can drift back to delivering it.

Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
This commit is contained in:
Yan Bondarenko committed 2026-10-18 23:16:45 +03:00
1 parent aed7fa896e
commit 89028b455e
3 files changed
+135 -1

No files matched your search

+89
View File
@@ -9,6 +9,10 @@ load test_helper
# scope: they are re-vendored by `make update`, so a bad severity there is
# fixed upstream or in patches/, not by editing the label.
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
# The alerts that exist only to be dropped, the one place `none` is right.
# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
# platform config does not have, so delivering it at any severity is noise.
ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
helm_actions_stripped() {
awk '
@@ -104,6 +108,14 @@ alert_severity_contract() {
break
fi
done
if [ "$severity" = "none" ]; then
for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
if [ "$blackholed" = "$alert" ]; then
accepted=1
break
fi
done
fi
if [ "$accepted" -eq 0 ]; then
echo "$file: $alert -> '$severity'" >&2
bad=$((bad + 1))
@@ -128,6 +140,83 @@ alert_severity_contract() {
alert_severity_contract
}
@test "InfoInhibitor is labelled for the blackhole route" {
rows=$(mktemp) || return 1
if ! alert_severity_rows > "$rows"; then
rm -f "$rows"
return 1
fi
tab=$(printf '\t')
found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
rm -f "$rows"
[ "$found" -gt 0 ]
[ "$delivered" -eq 0 ]
}
@test "severity none passes only on an alert that exists to be dropped" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
printf '%s\n' \
'apiVersion: operator.victoriametrics.com/v1beta1' \
'kind: VMRule' \
'spec:' \
' groups:' \
' - name: general' \
' rules:' \
' - alert: InfoInhibitor' \
' labels:' \
' severity: none' \
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
git -C "$fixture" init -q
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
inhibitor_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
printf '%s\n' \
'apiVersion: operator.victoriametrics.com/v1beta1' \
'kind: VMRule' \
'spec:' \
' groups:' \
' - name: silent' \
' rules:' \
' - alert: SilentAlert' \
' labels:' \
' severity: none' \
> "$fixture/packages/system/example/alerts/01-silent.yaml"
git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
silent_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
rm -rf "$fixture"
[ "$inhibitor_succeeded" -eq 0 ]
[ "$silent_succeeded" -ne 0 ]
}
@test "an alert that exists to be dropped still fails on a refused severity" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
printf '%s\n' \
'apiVersion: operator.victoriametrics.com/v1beta1' \
'kind: VMRule' \
'spec:' \
' groups:' \
' - name: general' \
' rules:' \
' - alert: InfoInhibitor' \
' labels:' \
' severity: info' \
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
git -C "$fixture" init -q
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
contract_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
rm -rf "$fixture"
[ "$contract_succeeded" -ne 0 ]
}
@test "a parser failure cannot leave the contract green" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
@@ -52,6 +52,6 @@ spec:
!= "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} ==
1
labels:
severity: major
severity: none
exported_instance: global
service: general.rules
@@ -0,0 +1,45 @@
suite: Alertmanager blackhole route
templates:
- templates/alerta/alerta.yaml
release:
name: monitoring
namespace: tenant-root
set:
_cluster:
root-host: example.org
_namespace:
host: tenant-root.example.org
ingress: tenant-root
# Rules labelled severity="none", InfoInhibitor among them, exist to be
# dropped, and Alerta answers that severity with a 500. Only the heartbeat
# route may come ahead of the blackhole, so no delivering route claims them.
tests:
- it: drops severity none ahead of every delivering route
asserts:
- documentSelector:
path: metadata.labels.app
value: alertmanager
matchRegex:
path: stringData["alertmanager.yaml"]
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
- it: keeps severity none out of email even when its alert name is listed
set:
alerta:
alerts:
email:
to: "oncall@example.org"
from: "alerts@example.org"
smarthost: "smtp.example.org:465"
alertnames:
- InfoInhibitor
asserts:
- documentSelector:
path: metadata.labels.app
value: alertmanager
matchRegex:
path: stringData["alertmanager.yaml"]
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'