diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats index e1315eb96..44f197000 100644 --- a/hack/alert-severity-contract.bats +++ b/hack/alert-severity-contract.bats @@ -9,6 +9,10 @@ load test_helper # scope: they are re-vendored by `make update`, so a bad severity there is # fixed upstream or in patches/, not by editing the label. ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown " +# The alerts that exist only to be dropped, the one place `none` is right. +# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the +# platform config does not have, so delivering it at any severity is noise. +ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor " helm_actions_stripped() { awk ' @@ -104,6 +108,14 @@ alert_severity_contract() { break fi done + if [ "$severity" = "none" ]; then + for blackholed in $ALERT_SEVERITY_BLACKHOLED; do + if [ "$blackholed" = "$alert" ]; then + accepted=1 + break + fi + done + fi if [ "$accepted" -eq 0 ]; then echo "$file: $alert -> '$severity'" >&2 bad=$((bad + 1)) @@ -128,6 +140,83 @@ alert_severity_contract() { alert_severity_contract } +@test "InfoInhibitor is labelled for the blackhole route" { + rows=$(mktemp) || return 1 + if ! alert_severity_rows > "$rows"; then + rm -f "$rows" + return 1 + fi + tab=$(printf '\t') + found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true) + delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true) + rm -f "$rows" + [ "$found" -gt 0 ] + [ "$delivered" -eq 0 ] +} + +@test "severity none passes only on an alert that exists to be dropped" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + inhibitor_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$? + + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: silent' \ + ' rules:' \ + ' - alert: SilentAlert' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/01-silent.yaml" + git -C "$fixture" add packages/system/example/alerts/01-silent.yaml + + silent_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$? + rm -rf "$fixture" + [ "$inhibitor_succeeded" -eq 0 ] + [ "$silent_succeeded" -ne 0 ] +} + +@test "an alert that exists to be dropped still fails on a refused severity" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: info' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -ne 0 ] +} + @test "a parser failure cannot leave the contract green" { fixture=$(mktemp -d) mkdir -p "$fixture/packages/system/example/alerts" diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml index 0ac83fe72..e713a4970 100644 --- a/packages/system/monitoring-agents/alerts/general.rules.yaml +++ b/packages/system/monitoring-agents/alerts/general.rules.yaml @@ -53,6 +53,6 @@ spec: != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 labels: - severity: major + severity: none exported_instance: global service: general.rules diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml new file mode 100644 index 000000000..d4e366180 --- /dev/null +++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml @@ -0,0 +1,51 @@ +suite: Alertmanager blackhole route +templates: + - templates/alerta/alerta.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + +# Rules labelled severity="none", InfoInhibitor among them, exist to be +# dropped, and Alerta answers that severity with a 500. Only the heartbeat +# route may come ahead of the blackhole, so no delivering route claims them. +tests: + - it: drops severity none ahead of every delivering route + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + + - it: keeps severity none out of email even when its alert name is listed + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + alertnames: + - InfoInhibitor + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '(?s)\n - receiver: blackhole\n matchers:\n - .severity="none".\n.*\n - receiver: email\n group_wait: 1m\n group_interval: 5m\n repeat_interval: .12h.\n matchers:\n - .alertname=~"\^\(InfoInhibitor\)\$".\n'