diff --git a/packages/extra/etcd/templates/prometheus-rules.yaml b/packages/extra/etcd/templates/prometheus-rules.yaml index a53706541..6f2be8590 100644 --- a/packages/extra/etcd/templates/prometheus-rules.yaml +++ b/packages/extra/etcd/templates/prometheus-rules.yaml @@ -10,7 +10,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'." expr: | - sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2) + sum(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 1) by (job) < ((count(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"}) by (job) + 1) / 2) for: 3m labels: severity: critical @@ -19,7 +19,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader." expr: | - etcd_server_has_leader{job=~".*etcd.*"} == 0 + etcd_server_has_leader{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == 0 for: 1m labels: severity: critical @@ -28,7 +28,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour." expr: | - rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3 + rate(etcd_server_leader_changes_seen_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 3 for: 15m labels: severity: warning @@ -37,9 +37,9 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) + 100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method) > 1 for: 10m labels: @@ -49,9 +49,9 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) + 100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method) > 5 for: 5m labels: @@ -61,7 +61,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) > 0.15 for: 10m labels: @@ -71,7 +71,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: @@ -81,7 +81,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + rate(etcd_server_proposals_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 5 for: 15m labels: severity: warning @@ -90,7 +90,7 @@ spec: annotations: summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01 + sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.01 for: 10m labels: severity: warning @@ -99,7 +99,7 @@ spec: annotations: summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05 + sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.05 for: 10m labels: severity: critical @@ -108,7 +108,7 @@ spec: annotations: summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow." expr: | - histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) + histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: @@ -120,10 +120,10 @@ spec: description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.' expr: | max without (endpoint) ( - sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0) + sum without (instance, pod) (up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 0) or count without (To) ( - sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[120s])) > 0.01 ) ) > 0 diff --git a/packages/extra/etcd/tests/namespace_rules_test.yaml b/packages/extra/etcd/tests/namespace_rules_test.yaml new file mode 100644 index 000000000..e972f3cd0 --- /dev/null +++ b/packages/extra/etcd/tests/namespace_rules_test.yaml @@ -0,0 +1,18 @@ +suite: etcd alerts stay inside their release namespace + +release: + name: etcd + namespace: tenant-scope + +templates: + - templates/prometheus-rules.yaml + +tests: + - it: scopes every alert query to this tenant's scrape job + asserts: + - matchRegex: + path: spec.groups[0].rules[*].expr + pattern: 'job="tenant-scope/etcd-pod-scrape"' + - notMatchRegex: + path: spec.groups[0].rules[*].expr + pattern: 'job=~' diff --git a/packages/system/monitoring-agents/alerts/etcd.yaml b/packages/system/monitoring-agents/alerts/etcd.yaml index 72c5ef7b2..48cdc84d9 100644 --- a/packages/system/monitoring-agents/alerts/etcd.yaml +++ b/packages/system/monitoring-agents/alerts/etcd.yaml @@ -116,8 +116,9 @@ spec: {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.' summary: etcd cluster member communication is slow. + # Tenant peer RTT is covered by the tenant etcd rules. expr: |- - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*", job!~"tenant-.*/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: diff --git a/packages/system/monitoring-agents/tests/etcd_rules_test.yaml b/packages/system/monitoring-agents/tests/etcd_rules_test.yaml new file mode 100644 index 000000000..78651297c --- /dev/null +++ b/packages/system/monitoring-agents/tests/etcd_rules_test.yaml @@ -0,0 +1,20 @@ +suite: tenant peer RTT is evaluated once + +templates: + - templates/alerts.yaml + +tests: + - it: leaves tenant peer RTT to the tenant rules + documentSelector: + path: metadata.name + value: alerts-etcd + asserts: + - matchRegex: + path: spec.groups[0].rules[7].expr + pattern: 'job!~"tenant-\.\*/etcd-pod-scrape"' + - notMatchRegex: + path: spec.groups[0].rules[9].expr + pattern: 'job!~' + - notMatchRegex: + path: spec.groups[0].rules[10].expr + pattern: 'job!~'