{ "uid": "gpu-quotas", "title": "GPU Quotas \u0026 Allocation", "tags": [ "gpu", "quotas" ], "timezone": "browser", "editable": true, "graphTooltip": 1, "time": { "from": "now-6h", "to": "now" }, "fiscalYearStartMonth": 0, "schemaVersion": 42, "panels": [ { "type": "row", "collapsed": false, "title": "Allocation overview", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [] }, { "type": "stat", "id": 2, "targets": [ { "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", "refId": "A" } ], "title": "GPU allocatable", "description": "Total GPU capacity the cluster can schedule to pods.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" } ] } }, "overrides": [] } }, { "type": "stat", "id": 3, "targets": [ { "expr": "cluster:gpu_count:allocated", "refId": "A" } ], "title": "GPU requested", "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 1, "color": "yellow" } ] } }, "overrides": [] } }, { "type": "gauge", "id": 4, "targets": [ { "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", "refId": "A" } ], "title": "Allocation ratio", "description": "Percentage of allocatable GPUs currently requested by pods.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, "repeatDirection": "h", "options": { "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto", "minVizWidth": 75, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "minVizHeight": 75, "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" }, { "value": 40, "color": "green" }, { "value": 80, "color": "yellow" }, { "value": 95, "color": "red" } ] } }, "overrides": [] } }, { "type": "stat", "id": 5, "targets": [ { "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", "refId": "A" } ], "title": "Pending pods (GPU)", "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 1, "color": "red" } ] } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Per namespace", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "panels": [] }, { "type": "bargauge", "id": 11, "targets": [ { "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}", "refId": "A" } ], "title": "GPU requested per namespace", "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "repeatDirection": "h", "options": { "displayMode": "gradient", "valueMode": "color", "namePlacement": "auto", "showUnfilled": true, "sizing": "auto", "minVizWidth": 8, "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "maxVizHeight": 300, "orientation": "horizontal" }, "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": 8, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] } }, "overrides": [] } }, { "type": "timeseries", "id": 12, "targets": [ { "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", "legendFormat": "Allocatable (total)", "refId": "A" }, { "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", "legendFormat": "Requested", "refId": "B" } ], "title": "GPU allocated over time", "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineInterpolation": "stepAfter", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [ { "matcher": { "id": "byName", "options": "Allocatable (total)" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] } }, { "type": "row", "collapsed": false, "title": "Pods with GPU", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 20, "panels": [] }, { "type": "table", "id": 21, "targets": [ { "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", "refId": "requested" }, { "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", "instant": true, "range": false, "format": "table", "refId": "limits" } ], "title": "Pods requesting GPU", "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 15 }, "repeatDirection": "h", "transformations": [ { "id": "joinByField", "options": { "byField": "pod", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Time 1": true, "Time 2": true, "__name__": true, "__name__ 1": true, "__name__ 2": true, "cluster": true, "cluster 1": true, "cluster 2": true, "container 2": true, "endpoint": true, "endpoint 1": true, "endpoint 2": true, "instance": true, "instance 1": true, "instance 2": true, "job": true, "job 1": true, "job 2": true, "namespace 2": true, "node 2": true, "prometheus": true, "prometheus 1": true, "prometheus 2": true, "resource": true, "resource 1": true, "resource 2": true, "service": true, "service 1": true, "service 2": true, "tenant": true, "tenant 1": true, "tenant 2": true, "tier": true, "tier 1": true, "tier 2": true, "uid": true, "uid 1": true, "uid 2": true, "unit": true, "unit 1": true, "unit 2": true }, "indexByName": { "Value #limits": 5, "Value #requested": 4, "container 1": 2, "namespace 1": 0, "node 1": 3, "phase": 6, "pod": 1 }, "renameByName": { "Value #limits": "Limit", "Value #requested": "Req", "container 1": "Container", "namespace 1": "Namespace", "node 1": "Node", "phase": "Status" } } } ], "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [] }, "cellHeight": "sm" }, "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, { "id": "mappings", "value": [ { "options": { "Failed": { "color": "red", "index": 2 }, "Pending": { "color": "orange", "index": 1 }, "Running": { "color": "green", "index": 0 } }, "type": "value" } ] } ] } ] } } ], "templating": { "list": [ { "type": "datasource", "name": "ds_prometheus", "label": "Prometheus", "skipUrlSync": false, "query": "prometheus", "current": { "selected": false, "text": "default", "value": "default" }, "multi": false, "allowCustomValue": true, "includeAll": false, "regex": "", "auto": false, "auto_min": "10s", "auto_count": 30 }, { "type": "query", "name": "namespace", "label": "Namespace", "skipUrlSync": false, "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "current": { "selected": true, "text": "All", "value": "$__all" }, "multi": true, "allowCustomValue": true, "refresh": 2, "sort": 1, "includeAll": true, "auto": false, "auto_min": "10s", "auto_count": 30 } ] }, "annotations": {} }