Files
cozystack/hack/alert-severity-contract.bats
T
Myasnikov Daniil cc942fed68 test(tests): run unit suites with Bats and retain POSIX coverage
The shell translator rewrites test bodies and loses Bats reporting.
The unit lane needs complete TAP results and JUnit reports without
changing how live-cluster suites run.

Sourced production helpers still need a dash pass, and unit test bodies
need nounset without exporting shell options into subprocesses.

Assisted-by: LLM
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
2026-10-07 11:04:00 +03:00

232 lines
7.5 KiB
Bash

#!/usr/bin/env bats
load test_helper
# Alerta's default alarm model, the set the webhook accepts. `none` is left out
# on purpose: the Alertmanager config routes severity="none" to a blackhole
# receiver, so such an alert is never delivered, which is the outcome this
# contract exists to catch. Rule files under packages/**/charts/** are out of
# scope: they are re-vendored by `make update`, so a bad severity there is
# fixed upstream or in patches/, not by editing the label.
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
helm_actions_stripped() {
awk '
function strip_comments(s, pre, rest) {
while (1) {
if (incomment) {
if (match(s, /\*\/[ \t]*-?[}][}]/)) { s = substr(s, RSTART + RLENGTH); incomment = 0 } else { return "" }
}
if (match(s, /[{][{]-?[ \t]*\/\*/)) {
pre = substr(s, 1, RSTART - 1); rest = substr(s, RSTART + RLENGTH)
if (match(rest, /\*\/[ \t]*-?[}][}]/)) { s = pre substr(rest, RSTART + RLENGTH) } else { incomment = 1; return pre }
} else { return s }
}
}
{
line = strip_comments($0)
if (incomment && line ~ /^[ \t]*$/) { next }
if (line ~ /^[ \t]*[{][{].*[}][}][ \t]*$/) { next }
gsub(/[{][{][^}]*[}][}]/, "TEMPLATED", line)
print line
}
' "$1"
}
alert_severity_rows() {
severity_root=${ALERT_SEVERITY_ROOT:-"$(dirname "${BATS_TEST_FILENAME:-$0}")/.."}
severity_files=$(mktemp) || return 1
severity_rows=$(mktemp) || {
rm -f "$severity_files"
return 1
}
severity_stripped=$(mktemp) || {
rm -f "$severity_files" "$severity_rows"
return 1
}
if ! (
cd "$severity_root" &&
git grep -Il -E '^[[:space:]]*kind:[[:space:]]*(PrometheusRule|VMRule)[[:space:]]*$' -- \
'packages/**/*.yaml' \
'packages/**/*.yml' \
':(exclude)packages/**/charts/**'
) > "$severity_files"; then
rm -f "$severity_files" "$severity_rows" "$severity_stripped"
return 1
fi
if ! : > "$severity_rows"; then
rm -f "$severity_files" "$severity_rows" "$severity_stripped"
return 1
fi
while IFS= read -r file; do
if ! (cd "$severity_root" && helm_actions_stripped "$file" > "$severity_stripped" && FILE="$file" yq 'select(.kind == "PrometheusRule" or .kind == "VMRule") | .spec.groups[]? | .rules[]? | select(has("alert")) | [strenv(FILE), .alert, ((.labels.severity // "<missing>") | tostring)] | @tsv' "$severity_stripped") >> "$severity_rows"; then
echo "$file: failed to parse alert rules" >&2
rm -f "$severity_files" "$severity_rows" "$severity_stripped"
return 1
fi
done < "$severity_files"
if ! sed '/^[[:space:]]*$/d' "$severity_rows"; then
rm -f "$severity_files" "$severity_rows" "$severity_stripped"
return 1
fi
rm -f "$severity_files" "$severity_rows" "$severity_stripped"
}
alert_severity_contract() {
rows=$(mktemp) || return 1
if ! alert_severity_rows > "$rows"; then
rm -f "$rows"
return 1
fi
bad=0
tab=$(printf '\t')
while IFS="$tab" read -r file alert severity; do
accepted=0
for allowed in $ALERT_SEVERITY_ALLOWED; do
if [ "$allowed" = "$severity" ]; then
accepted=1
break
fi
done
if [ "$accepted" -eq 0 ]; then
echo "$file: $alert -> '$severity'" >&2
bad=$((bad + 1))
fi
done < "$rows"
rm -f "$rows"
[ "$bad" -eq 0 ] || return 1
}
@test "alert rules are discovered" {
rows=$(mktemp) || return 1
if ! alert_severity_rows > "$rows"; then
rm -f "$rows"
return 1
fi
count=$(wc -l < "$rows" | tr -d ' ')
rm -f "$rows"
[ "$count" -gt 0 ]
}
@test "every alert rule carries a severity the alert sink accepts" {
alert_severity_contract
}
@test "a parser failure cannot leave the contract green" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
printf '%s\n' \
'apiVersion: operator.victoriametrics.com/v1beta1' \
'kind: VMRule' \
'spec:' \
' groups:' \
' - name: valid' \
' rules:' \
' - alert: ValidAlert' \
' labels:' \
' severity: warning' \
> "$fixture/packages/system/example/alerts/00-valid.yaml"
printf '%s\n' \
'apiVersion: operator.victoriametrics.com/v1beta1' \
'kind: VMRule' \
'spec: [' \
> "$fixture/packages/system/example/alerts/99-malformed.yaml"
git -C "$fixture" init -q
git -C "$fixture" add packages/system/example/alerts/00-valid.yaml packages/system/example/alerts/99-malformed.yaml
parser_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_rows >/dev/null 2>&1 || parser_succeeded=$?
rm -rf "$fixture"
[ "$parser_succeeded" -ne 0 ]
}
@test "template rule sources cannot bypass the severity contract" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/extra/example/templates"
printf '%s\n' \
'apiVersion: monitoring.coreos.com/v1' \
'kind: PrometheusRule' \
'spec:' \
' groups:' \
' - name: template' \
' rules:' \
' - alert: InvalidTemplateSeverity' \
' labels:' \
' severity: warn' \
> "$fixture/packages/extra/example/templates/prometheus-rules.yaml"
git -C "$fixture" init -q
git -C "$fixture" add packages/extra/example/templates/prometheus-rules.yaml
contract_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
rm -rf "$fixture"
[ "$contract_succeeded" -ne 0 ]
}
@test "helm template actions do not hide a rule from the contract" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/templates"
printf '%s\n' \
'{{- if .Values.alerts.enabled }}' \
'{{- /*' \
'Rules gated on a value, behind a comment that spans lines.' \
'*/}}' \
'apiVersion: monitoring.coreos.com/v1' \
'kind: PrometheusRule' \
'metadata:' \
' name: example' \
' namespace: {{ .Release.Namespace }}' \
'spec:' \
' groups:' \
' - name: example' \
' rules:' \
' {{- /* one-line comment */}}' \
' - alert: TemplatedRuleWithBadSeverity' \
' expr: up == 0' \
' labels:' \
' severity: warn' \
' annotations:' \
' description: {{ "{{ $labels.instance }}" }} is down' \
'{{- end }}' \
> "$fixture/packages/system/example/templates/alerts.yaml"
git -C "$fixture" init -q
git -C "$fixture" add packages/system/example/templates/alerts.yaml
rows=$(mktemp)
parser_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_rows > "$rows" 2>/dev/null || parser_succeeded=$?
found=$(grep -c 'TemplatedRuleWithBadSeverity' "$rows" || true)
contract_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
rm -rf "$fixture" "$rows"
[ "$parser_succeeded" -eq 0 ]
[ "$found" -eq 1 ]
[ "$contract_succeeded" -ne 0 ]
}
@test "two accepted words in one severity are rejected" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
printf '%s\n' \
'apiVersion: operator.victoriametrics.com/v1beta1' \
'kind: VMRule' \
'spec:' \
' groups:' \
' - name: joined' \
' rules:' \
' - alert: JoinedSeverity' \
' labels:' \
' severity: critical major' \
> "$fixture/packages/system/example/alerts/00-joined.yaml"
git -C "$fixture" init -q
git -C "$fixture" add packages/system/example/alerts/00-joined.yaml
contract_succeeded=0
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
rm -rf "$fixture"
[ "$contract_succeeded" -ne 0 ]
}