From 5632596fe84fa358191c62f5f316df62f32c865c Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:38:55 +0300 Subject: [PATCH] feat(monitoring): alert when pods of one workload share a node Two etcd members, two Redis Sentinels or both replicas of a storage tier can end up on one node through a soft spread rule, a drain, or a placement made before a spread rule existed. Nothing reported it, and the scheduler does not move running pods, so it stays like that until a pod is recreated. One node failure then costs the quorum or every copy at once. etcd-operator owns each member pod by its own EtcdMember, so members are grouped by the EtcdCluster name their names start with. Owner and node are read from kube_pod_info because kube_pod_owner and kube_pod_status_phase get the node of the kube-state-metrics pod as a target label. Nodes with a NoSchedule or NoExecute taint do not count, which covers cordoned, not ready and unreachable nodes. The alert waits an hour, longer than a replacement pod runs next to its terminating predecessor or a node takes to drain and come back. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../workload-placement-alerts/unittest.yaml | 572 ++++++++++++++++++ hack/workload-placement-alerts.bats | 26 + .../alerts/workload-placement.yaml | 81 +++ 3 files changed, 679 insertions(+) create mode 100644 hack/testdata/workload-placement-alerts/unittest.yaml create mode 100644 hack/workload-placement-alerts.bats create mode 100644 packages/system/monitoring-agents/alerts/workload-placement.yaml diff --git a/hack/testdata/workload-placement-alerts/unittest.yaml b/hack/testdata/workload-placement-alerts/unittest.yaml new file mode 100644 index 000000000..2565a8241 --- /dev/null +++ b/hack/testdata/workload-placement-alerts/unittest.yaml @@ -0,0 +1,572 @@ +rule_files: + - rules.yaml + +evaluation_interval: 15s + +tests: + - name: two of three StatefulSet pods on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 59m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StatefulSet + owner_name: db + node: node-a + severity: warning + exported_instance: tenant-a/db + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StatefulSet tenant-a/db are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: StatefulSet pods spread over three nodes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-c", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: five ReplicaSet pods on three nodes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", node="node-c", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: DaemonSet surge pod next to the old one + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", node="node-b", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: two pods of one Job on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: etcd members of one EtcdCluster on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-x7k2p"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-q9z4m"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-b5n8w"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-0"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-1"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-2"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: EtcdCluster + owner_name: etcd + node: node-a + severity: warning + exported_instance: tenant-a/etcd + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of EtcdCluster tenant-a/etcd are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-b + owner_kind: EtcdCluster + owner_name: etcd + node: node-c + severity: warning + exported_instance: tenant-b/etcd + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of EtcdCluster tenant-b/etcd are on node node-c in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: replacement pod next to its terminating predecessor for twenty minutes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x30 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '_x10 1x140' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}' + values: '1x30 stale' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}' + values: '_x10 1x140' + alert_rule_test: + - eval_time: 29m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 75m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: the same overlap held past the hour + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x100 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '_x10 1x140' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}' + values: '1x100 stale' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}' + values: '_x10 1x140' + alert_rule_test: + - eval_time: 69m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 71m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: ReplicaSet + owner_name: web-6b8f4 + node: node-b + severity: warning + exported_instance: tenant-a/web-6b8f4 + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of ReplicaSet tenant-a/web-6b8f4 are on node node-b in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - eval_time: 105m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: a cordoned node does not count until it is uncordoned + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x30 0x120' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unschedulable", value="", effect="NoSchedule"}' + values: '1x30 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 90m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 92m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StatefulSet + owner_name: kv + node: node-a + severity: warning + exported_instance: tenant-a/kv + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StatefulSet tenant-a/kv are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: a node that is down does not count + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '0x150' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoSchedule"}' + values: '1x150' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoExecute"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: pods that are not running do not count + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Running", node="node-c"}' + values: '0x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Failed", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: one pod each in two clusters that share workload and node names + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", phase="Running", node="node-b"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: ReplicaSet, CNPG Cluster and StrimziPodSet pods on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", node="node-b", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", node="node-a", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", node="node-a", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: ReplicaSet + owner_name: rfs-cache-6f7d8b9c5d + node: node-a + severity: warning + exported_instance: tenant-a/rfs-cache-6f7d8b9c5d + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of ReplicaSet tenant-a/rfs-cache-6f7d8b9c5d are on node node-a in cluster + cozystack, although the cluster has a schedulable node for each of its pods. Losing this + node stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: Cluster + owner_name: postgres-db + node: node-b + severity: warning + exported_instance: tenant-a/postgres-db + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of Cluster tenant-a/postgres-db are on node node-b in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StrimziPodSet + owner_name: kafka-kafka + node: node-c + severity: warning + exported_instance: tenant-a/kafka-kafka + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StrimziPodSet tenant-a/kafka-kafka are on node node-c in cluster + cozystack, although the cluster has a schedulable node for each of its pods. Losing + this node stops these pods together, which can cost a quorum-based workload its quorum. + The scheduler does not move running pods, so they stay together until they are + recreated. diff --git a/hack/workload-placement-alerts.bats b/hack/workload-placement-alerts.bats new file mode 100644 index 000000000..3f7b9b961 --- /dev/null +++ b/hack/workload-placement-alerts.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1" + +@test "workload placement alerts pass their vmalert-tool cases" { + docker info >/dev/null 2>&1 || { + echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2 + return 1 + } + work=$(mktemp -d) + rc=0 + yq '.spec' "$REPO_ROOT/packages/system/monitoring-agents/alerts/workload-placement.yaml" > "$work/rules.yaml" || rc=$? + cp "$REPO_ROOT/hack/testdata/workload-placement-alerts/unittest.yaml" "$work/unittest.yaml" || rc=$? + if [ "$rc" -eq 0 ]; then + docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \ + unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$? + cat "$work/out" + fi + passed=$(grep -cx 'SUCCESS' "$work/out" 2>/dev/null || true) + rm -rf "$work" + [ "$rc" -eq 0 ] + [ "$passed" -eq 1 ] +} diff --git a/packages/system/monitoring-agents/alerts/workload-placement.yaml b/packages/system/monitoring-agents/alerts/workload-placement.yaml new file mode 100644 index 000000000..3ae9c8f50 --- /dev/null +++ b/packages/system/monitoring-agents/alerts/workload-placement.yaml @@ -0,0 +1,81 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: alerts-workload-placement +spec: + groups: + - name: workload-placement + params: {} + rules: + - alert: PodReplicasColocated + expr: |- + count by (cluster, namespace, owner_kind, owner_name, node) ( + label_replace( + label_replace( + ( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"} + ) + or + label_replace( + label_replace( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"} + ), + "created_by_name", "$1", "created_by_name", "(.+)-[^-]+" + ), + "created_by_kind", "EtcdCluster", "", "" + ) + ) + and on (cluster, namespace, pod) + kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1, + "owner_kind", "$1", "created_by_kind", "(.+)" + ), + "owner_name", "$1", "created_by_name", "(.+)" + ) + ) > 1 + and on (cluster, namespace, owner_kind, owner_name) + count by (cluster, namespace, owner_kind, owner_name) ( + label_replace( + label_replace( + ( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"} + ) + or + label_replace( + label_replace( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"} + ), + "created_by_name", "$1", "created_by_name", "(.+)-[^-]+" + ), + "created_by_kind", "EtcdCluster", "", "" + ) + ) + and on (cluster, namespace, pod) + kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1, + "owner_kind", "$1", "created_by_kind", "(.+)" + ), + "owner_name", "$1", "created_by_name", "(.+)" + ) + ) + <= on (cluster) group_left () + count by (cluster) ( + max by (cluster, node) (kube_node_info{job="kube-state-metrics"}) + unless on (cluster, node) + kube_node_spec_taint{job="kube-state-metrics", effect=~"NoSchedule|NoExecute"} + ) + for: 1h + labels: + severity: warning + exported_instance: '{{ $labels.namespace }}/{{ $labels.owner_name }}' + service: workload-placement + annotations: + summary: Pods of one workload share a node. + description: >- + {{ $value }} running pods of {{ $labels.owner_kind }} {{ $labels.namespace }}/{{ $labels.owner_name }} + are on node {{ $labels.node }} in cluster {{ $labels.cluster }}, although the cluster has a + schedulable node for each of its pods. Losing this node stops these pods together, which can cost + a quorum-based workload its quorum. The scheduler does not move running pods, so they stay + together until they are recreated.