fix(piraeus-operator): keep the LINSTOR error-report alerts quiet when the controller pod moves

linstor_error_reports_count comes from the LINSTOR controller, so its
series carry the controller pod's instance and controller_node labels.
When the pod moves to another node, or an external label of the scrape
changes, the existing report counts come back as new series.
VictoriaMetrics' increase() counts the first samples of a series from
zero when they are small, and report counts are small, so
linstorSatelliteErrorRate fires for every node with an old report and
keeps firing for 15 minutes. The up guard matches on instance and lets
the new series through.

The controller also leaves a satellite out of the metric while it is
not connected, so a satellite that comes back after more than five
minutes has its old count counted again in the same way.

Keyed on hostname, a moved controller or a changed label continues the
same series, and holding the last value for an hour bridges a
disconnect or a slow controller restart. The up guard matches on()
because the result no longer has an instance label. The controller
rule takes the same form. Its hostname is the pod name and a new pod
starts with no reports, but a label change hit it the same way.

Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
This commit is contained in:
Yan Bondarenko committed 2026-10-19 19:41:22 +03:00
1 parent aed7fa896e
commit 8502629995
3 files changed
+233 -3

No files matched your search

+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bats
# Replays the LINSTOR error-report alerts from
# packages/system/piraeus-operator/alerts/piraeus-datastore.yaml through
# vmalert-tool, pinned to the VictoriaMetrics release that the
# victoria-metrics-operator deploys by default. Its increase() counts the first
# samples of a series as new when they are small, and the report counts are
# small, so the cases pin that a series which starts over after a controller
# move, a label change or a disconnect does not report old error reports again.
# Only pulling the image touches the network.
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
@test "LINSTOR error-report alerts pass their vmalert-tool cases" {
docker info >/dev/null 2>&1 || {
echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
return 1
}
work=$(mktemp -d)
yq '.spec' "$REPO_ROOT/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml" > "$work/rules.yaml"
cp "$REPO_ROOT/hack/testdata/linstor-error-report-alerts/unittest.yaml" "$work/unittest.yaml"
rc=0
docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
cat "$work/out"
passed=$(grep -cx 'SUCCESS' "$work/out" || true)
rm -rf "$work"
[ "$rc" -eq 0 ]
[ "$passed" -eq 1 ]
}
+198
View File
@@ -0,0 +1,198 @@
rule_files:
- rules.yaml
evaluation_interval: 15s
tests:
- name: controller pod moves to another node
interval: 30s
input_series:
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a"}'
values: '1x38 0 stale'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-a", module="SATELLITE"}'
values: '1x38 stale'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-b", module="SATELLITE"}'
values: '2x38 stale'
- series: 'up{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b"}'
values: '_x41 1x79'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-a", module="SATELLITE"}'
values: '_x41 1x58 2x20'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-b", module="SATELLITE"}'
values: '_x41 2x79'
alert_rule_test:
- eval_time: 21m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 28m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 35m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 45m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 51m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
exp_alerts:
- exp_labels:
hostname: node-a
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-a" to inspect the reports.
- name: an external label of the scrape changes
interval: 30s
input_series:
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack"}'
values: '1x39 stale'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="node-a", module="SATELLITE"}'
values: '1x39 stale'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="linstor-controller-abc", module="CONTROLLER"}'
values: '1x39 stale'
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a"}'
values: '_x41 1x79'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="node-a", module="SATELLITE"}'
values: '_x41 1x79'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="linstor-controller-abc", module="CONTROLLER"}'
values: '_x41 1x79'
alert_rule_test:
- eval_time: 21m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 30m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 21m
groupname: linstor.rules
alertname: linstorControllerErrorRate
- eval_time: 30m
groupname: linstor.rules
alertname: linstorControllerErrorRate
- name: a satellite drops out of the metric while it is disconnected
interval: 30s
input_series:
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
values: '1x220'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
values: '1x38 stale _x80 1x100'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}'
values: '2x220'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}'
values: '1x38 stale _x80 2x100'
alert_rule_test:
- eval_time: 61m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
exp_alerts:
- exp_labels:
hostname: node-c
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-c" to inspect the reports.
- eval_time: 70m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
exp_alerts:
- exp_labels:
hostname: node-c
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-c" to inspect the reports.
- eval_time: 76m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- name: a new report fires for the 15 minute window
interval: 30s
input_series:
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
values: '1x120'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
values: '1x39 2x80'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}'
values: '2x120'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}'
values: '0x39 1x80'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="linstor-controller-abc", module="CONTROLLER"}'
values: '0x39 1x80'
alert_rule_test:
- eval_time: 21m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
exp_alerts:
- exp_labels:
hostname: node-a
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-a" to inspect the reports.
- exp_labels:
hostname: node-c
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-c" to inspect the reports.
- eval_time: 34m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
exp_alerts:
- exp_labels:
hostname: node-a
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-a" to inspect the reports.
- exp_labels:
hostname: node-c
severity: warning
exp_annotations:
description: |
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --nodes node-c" to inspect the reports.
- eval_time: 36m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 21m
groupname: linstor.rules
alertname: linstorControllerErrorRate
exp_alerts:
- exp_labels:
hostname: linstor-controller-abc
severity: warning
exp_annotations:
description: |
LINSTOR Controller reports 1 errors in the last 15 minutes.
Use "linstor error-reports list --since 15minutes" to see them.
- eval_time: 36m
groupname: linstor.rules
alertname: linstorControllerErrorRate
- name: a failed controller scrape still holds the alert back
interval: 30s
input_series:
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
values: '1x19 0x140 1x60'
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
values: '1x19 stale _x140 1x60'
alert_rule_test:
- eval_time: 85m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 93m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
- eval_time: 100m
groupname: linstor.rules
alertname: linstorSatelliteErrorRate
@@ -29,8 +29,8 @@ spec:
LINSTOR Satellite "{{ $labels.hostname }}" reports {{ $value }} errors in the last 15 minutes.
Use "linstor error-reports list --nodes {{ $labels.hostname }}" to inspect the reports.
expr: |
increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0
and on(instance, job)
increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="SATELLITE"}[1h])))[15m:1m]) > 0
and on()
min_over_time(up{job="linstor-controller"}[15m]) == 1
labels:
severity: warning
@@ -39,7 +39,7 @@ spec:
description: |
LINSTOR Controller reports {{ $value }} errors in the last 15 minutes.
Use "linstor error-reports list --since 15minutes" to see them.
expr: increase(linstor_error_reports_count{module="CONTROLLER"}[15m]) > 0
expr: increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="CONTROLLER"}[1h])))[15m:1m]) > 0
labels:
severity: warning
- alert: linstorSatelliteNotOnline