mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(monitoring): route InfoInhibitor to the blackhole receiver
InfoInhibitor exists to be the source side of an Alertmanager inhibit rule and, as its own description says, to go to a null receiver. The platform Alertmanager config has no inhibit rule, and the rule has carried severity major since it was imported alongside Alerta, so Alerta shows it one level below critical. Alerta's Telegram and Slack plugins forward it by default, and a notify_email label inherited from the info alert gets it emailed too. It fires whenever a rule labelled severity="info" is active with no warning or critical alert in the same namespace and cluster. The platform's own rules no longer use info, but rules it does not write still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt generates KubeVirtVMGuestMemoryAvailableLow, and operators add their own. Each of them turns into a major InfoInhibitor in Alerta. none is the severity upstream kube-prometheus gives this rule, and the platform config already sends severity="none" to the blackhole ahead of every route that delivers. The severity contract refuses none because such an alert is never delivered, so InfoInhibitor is named as the one alert for which that is the point. Tests hold both ends, so neither the label nor the route order can drift back to delivering it. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
This commit is contained in:
1 parent
aed7fa896e
commit
89028b455e
3 files changed
+135
-1
No files matched your search
@@ -9,6 +9,10 @@ load test_helper
|
||||
# scope: they are re-vendored by `make update`, so a bad severity there is
|
||||
# fixed upstream or in patches/, not by editing the label.
|
||||
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
|
||||
# The alerts that exist only to be dropped, the one place `none` is right.
|
||||
# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
|
||||
# platform config does not have, so delivering it at any severity is noise.
|
||||
ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
|
||||
|
||||
helm_actions_stripped() {
|
||||
awk '
|
||||
@@ -104,6 +108,14 @@ alert_severity_contract() {
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$severity" = "none" ]; then
|
||||
for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
|
||||
if [ "$blackholed" = "$alert" ]; then
|
||||
accepted=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ "$accepted" -eq 0 ]; then
|
||||
echo "$file: $alert -> '$severity'" >&2
|
||||
bad=$((bad + 1))
|
||||
@@ -128,6 +140,83 @@ alert_severity_contract() {
|
||||
alert_severity_contract
|
||||
}
|
||||
|
||||
@test "InfoInhibitor is labelled for the blackhole route" {
|
||||
rows=$(mktemp) || return 1
|
||||
if ! alert_severity_rows > "$rows"; then
|
||||
rm -f "$rows"
|
||||
return 1
|
||||
fi
|
||||
tab=$(printf '\t')
|
||||
found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
|
||||
delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
|
||||
rm -f "$rows"
|
||||
[ "$found" -gt 0 ]
|
||||
[ "$delivered" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "severity none passes only on an alert that exists to be dropped" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: general' \
|
||||
' rules:' \
|
||||
' - alert: InfoInhibitor' \
|
||||
' labels:' \
|
||||
' severity: none' \
|
||||
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
|
||||
git -C "$fixture" init -q
|
||||
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
|
||||
|
||||
inhibitor_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
|
||||
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: silent' \
|
||||
' rules:' \
|
||||
' - alert: SilentAlert' \
|
||||
' labels:' \
|
||||
' severity: none' \
|
||||
> "$fixture/packages/system/example/alerts/01-silent.yaml"
|
||||
git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
|
||||
|
||||
silent_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
|
||||
rm -rf "$fixture"
|
||||
[ "$inhibitor_succeeded" -eq 0 ]
|
||||
[ "$silent_succeeded" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "an alert that exists to be dropped still fails on a refused severity" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: general' \
|
||||
' rules:' \
|
||||
' - alert: InfoInhibitor' \
|
||||
' labels:' \
|
||||
' severity: info' \
|
||||
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
|
||||
git -C "$fixture" init -q
|
||||
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
|
||||
|
||||
contract_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
|
||||
rm -rf "$fixture"
|
||||
[ "$contract_succeeded" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "a parser failure cannot leave the contract green" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
|
||||
@@ -52,6 +52,6 @@ spec:
|
||||
!= "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} ==
|
||||
1
|
||||
labels:
|
||||
severity: major
|
||||
severity: none
|
||||
exported_instance: global
|
||||
service: general.rules
|
||||
@@ -0,0 +1,45 @@
|
||||
suite: Alertmanager blackhole route
|
||||
templates:
|
||||
- templates/alerta/alerta.yaml
|
||||
|
||||
release:
|
||||
name: monitoring
|
||||
namespace: tenant-root
|
||||
|
||||
set:
|
||||
_cluster:
|
||||
root-host: example.org
|
||||
_namespace:
|
||||
host: tenant-root.example.org
|
||||
ingress: tenant-root
|
||||
|
||||
# Rules labelled severity="none", InfoInhibitor among them, exist to be
|
||||
# dropped, and Alerta answers that severity with a 500. Only the heartbeat
|
||||
# route may come ahead of the blackhole, so no delivering route claims them.
|
||||
tests:
|
||||
- it: drops severity none ahead of every delivering route
|
||||
asserts:
|
||||
- documentSelector:
|
||||
path: metadata.labels.app
|
||||
value: alertmanager
|
||||
matchRegex:
|
||||
path: stringData["alertmanager.yaml"]
|
||||
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
|
||||
|
||||
- it: keeps severity none out of email even when its alert name is listed
|
||||
set:
|
||||
alerta:
|
||||
alerts:
|
||||
email:
|
||||
to: "oncall@example.org"
|
||||
from: "alerts@example.org"
|
||||
smarthost: "smtp.example.org:465"
|
||||
alertnames:
|
||||
- InfoInhibitor
|
||||
asserts:
|
||||
- documentSelector:
|
||||
path: metadata.labels.app
|
||||
value: alertmanager
|
||||
matchRegex:
|
||||
path: stringData["alertmanager.yaml"]
|
||||
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
|
||||
Reference in new issue
Block a user