mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(etcd): scope tenant alerts to their scrape job (#5183)
## What this PR does scopes every tenant etcd alert to its release scrape job. previously each release evaluated every tenant, so one failing peer could produce alerts from unrelated releases. tenant peer RTT is evaluated by the tenant rules, while the remaining global etcd checks stay in place. validation: both affected chart test runs pass, 46 tests in 17 suites. both charts render. an expression-only simulated leader-loss check against existing metrics selected 18 members across six namespaces before scoping and only the three intended members afterwards. thresholds are unchanged. ### Screenshots Not a UI change. ### Downstream repositories - [x] No downstream repository is affected by this change - [ ] [cozystack/website](https://github.com/cozystack/website) - follow-up: - [ ] [cozystack/terraform-provider-cozystack](https://github.com/cozystack/terraform-provider-cozystack) - follow-up: - [ ] [cozystack/ansible-cozystack](https://github.com/cozystack/ansible-cozystack) - follow-up: - [ ] [cozystack/ccp](https://github.com/cozystack/ccp) - follow-up: - [ ] [cozystack/talm](https://github.com/cozystack/talm) - follow-up: - [ ] [cozystack/cozyhr](https://github.com/cozystack/cozyhr) - follow-up: - [ ] [cozystack/cozy-proxy](https://github.com/cozystack/cozy-proxy) - follow-up: - [ ] [cozystack/cozystack-telemetry-server](https://github.com/cozystack/cozystack-telemetry-server) - follow-up: - [ ] [cozystack/external-apps-example](https://github.com/cozystack/external-apps-example) - follow-up: - [ ] [cozystack/examples](https://github.com/cozystack/examples) - follow-up: - [ ] [cozystack/community](https://github.com/cozystack/community) - follow-up: no API fields, metric names, package interfaces or documented workarounds change. ### Release note ```release-note fix(etcd): scope tenant etcd alerts to their release and avoid duplicate tenant peer RTT alerts from the global rules ``` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> <!-- coderabbit-ai-generated:v1 --> ## Summary by CodeRabbit * **Bug Fixes** * Etcd alerts now match metrics from the exact etcd scrape job, including slow HTTP requests, reducing the chance of unrelated metrics triggering alerts. * The peer communication latency alert excludes tenant etcd scrape jobs, which are covered by tenant-specific rules. * **Tests** * Added checks to validate etcd alert job matching and tenant-job exclusions. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
4 files changed
+55
-16
No files matched your search
@@ -10,7 +10,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'."
|
||||
expr: |
|
||||
sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
|
||||
sum(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 1) by (job) < ((count(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"}) by (job) + 1) / 2)
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
@@ -19,7 +19,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader."
|
||||
expr: |
|
||||
etcd_server_has_leader{job=~".*etcd.*"} == 0
|
||||
etcd_server_has_leader{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
@@ -28,7 +28,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour."
|
||||
expr: |
|
||||
rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
|
||||
rate(etcd_server_leader_changes_seen_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 3
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
@@ -37,9 +37,9 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
/
|
||||
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
> 1
|
||||
for: 10m
|
||||
labels:
|
||||
@@ -49,9 +49,9 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
/
|
||||
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method)
|
||||
> 5
|
||||
for: 5m
|
||||
labels:
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
|
||||
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
@@ -71,7 +71,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
|
||||
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m]))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
@@ -81,7 +81,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
|
||||
rate(etcd_server_proposals_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
@@ -90,7 +90,7 @@ spec:
|
||||
annotations:
|
||||
summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01
|
||||
sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.01
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
@@ -99,7 +99,7 @@ spec:
|
||||
annotations:
|
||||
summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
|
||||
expr: |
|
||||
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05
|
||||
sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
@@ -108,7 +108,7 @@ spec:
|
||||
annotations:
|
||||
summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow."
|
||||
expr: |
|
||||
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
|
||||
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m]))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
@@ -120,10 +120,10 @@ spec:
|
||||
description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.'
|
||||
expr: |
|
||||
max without (endpoint) (
|
||||
sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
|
||||
sum without (instance, pod) (up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 0)
|
||||
or
|
||||
count without (To) (
|
||||
sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
|
||||
sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[120s])) > 0.01
|
||||
)
|
||||
)
|
||||
> 0
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
suite: etcd alerts stay inside their release namespace
|
||||
|
||||
release:
|
||||
name: etcd
|
||||
namespace: tenant-scope
|
||||
|
||||
templates:
|
||||
- templates/prometheus-rules.yaml
|
||||
|
||||
tests:
|
||||
- it: scopes every alert query to this tenant's scrape job
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.groups[0].rules[*].expr
|
||||
pattern: 'job="tenant-scope/etcd-pod-scrape"'
|
||||
- notMatchRegex:
|
||||
path: spec.groups[0].rules[*].expr
|
||||
pattern: 'job=~'
|
||||
@@ -116,8 +116,9 @@ spec:
|
||||
{{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance
|
||||
}}.'
|
||||
summary: etcd cluster member communication is slow.
|
||||
# Tenant peer RTT is covered by the tenant etcd rules.
|
||||
expr: |-
|
||||
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
|
||||
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*", job!~"tenant-.*/etcd-pod-scrape"}[5m]))
|
||||
> 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
suite: tenant peer RTT is evaluated once
|
||||
|
||||
templates:
|
||||
- templates/alerts.yaml
|
||||
|
||||
tests:
|
||||
- it: leaves tenant peer RTT to the tenant rules
|
||||
documentSelector:
|
||||
path: metadata.name
|
||||
value: alerts-etcd
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: spec.groups[0].rules[7].expr
|
||||
pattern: 'job!~"tenant-\.\*/etcd-pod-scrape"'
|
||||
- notMatchRegex:
|
||||
path: spec.groups[0].rules[9].expr
|
||||
pattern: 'job!~'
|
||||
- notMatchRegex:
|
||||
path: spec.groups[0].rules[10].expr
|
||||
pattern: 'job!~'
|
||||
Reference in new issue
Block a user