{ "uid": "gpu-fleet", "title": "GPU Fleet Overview", "description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.", "tags": [ "gpu", "fleet", "admin" ], "timezone": "browser", "editable": true, "graphTooltip": 1, "time": { "from": "now-6h", "to": "now" }, "fiscalYearStartMonth": 0, "schemaVersion": 42, "panels": [ { "type": "row", "collapsed": false, "title": "Cluster snapshot", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [] }, { "type": "stat", "id": 2, "targets": [ { "expr": "cluster:gpu_count:total", "refId": "A" } ], "title": "Total GPUs", "description": "Physical GPUs discovered by DCGM across all nodes.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" } ] } }, "overrides": [] } }, { "type": "stat", "id": 3, "targets": [ { "expr": "cluster:gpu_count:allocated", "refId": "A" } ], "title": "Allocated", "description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" }, { "value": 1, "color": "green" } ] } }, "overrides": [] } }, { "type": "stat", "id": 4, "targets": [ { "expr": "cluster:gpu_count:free", "refId": "A" } ], "title": "Free", "description": "Unallocated capacity available for new tenants.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "yellow" }, { "value": 2, "color": "green" } ] } }, "overrides": [] } }, { "type": "stat", "id": 5, "targets": [ { "expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))", "refId": "A" } ], "title": "Nodes with GPU", "description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" } ] } }, "overrides": [] } }, { "type": "stat", "id": 6, "targets": [ { "expr": "cluster:gpu_util:avg", "refId": "A" } ], "title": "Average utilization", "description": "Legacy NVML utilization across all tenant GPUs.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" }, { "value": 30, "color": "green" }, { "value": 80, "color": "orange" } ] } }, "overrides": [] } }, { "type": "stat", "id": 7, "targets": [ { "expr": "cluster:gpu_power_watts:sum", "refId": "A" } ], "title": "Total power", "description": "Live power draw summed across the fleet.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "repeatDirection": "h", "options": { "graphMode": "area", "colorMode": "value", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "watt", "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" } ] } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Per-node GPU activity", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "panels": [] }, { "type": "bargauge", "id": 11, "targets": [ { "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", "legendFormat": "{{Hostname}}", "refId": "A" } ], "title": "GPU utilization per node", "description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "repeatDirection": "h", "options": { "displayMode": "gradient", "valueMode": "color", "namePlacement": "auto", "showUnfilled": true, "sizing": "auto", "minVizWidth": 8, "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "maxVizHeight": 300, "orientation": "horizontal" }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "blue" }, { "value": 30, "color": "green" }, { "value": 80, "color": "orange" } ] } }, "overrides": [] } }, { "type": "bargauge", "id": 12, "targets": [ { "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100", "legendFormat": "{{Hostname}}", "refId": "A" } ], "title": "Power draw per node (% of TDP)", "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "repeatDirection": "h", "options": { "displayMode": "gradient", "valueMode": "color", "namePlacement": "auto", "showUnfilled": true, "sizing": "auto", "minVizWidth": 8, "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "maxVizHeight": 300, "orientation": "horizontal" }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 60, "color": "yellow" }, { "value": 80, "color": "orange" } ] } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Usage trends", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 20, "panels": [] }, { "type": "timeseries", "id": 21, "targets": [ { "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", "legendFormat": "{{Hostname}}", "refId": "A" } ], "title": "GPU utilization per node (trend)", "description": "Per-node NVML utilization over time — shows shifting workload across hosts.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 20, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 22, "targets": [ { "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", "legendFormat": "{{Hostname}}", "refId": "A" } ], "title": "Power draw per node (trend)", "description": "Per-node GPU power consumption over time.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "watt", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 20, "showPoints": "never" } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Health", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 30, "panels": [] }, { "type": "stat", "id": 31, "targets": [ { "expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)", "refId": "A" } ], "title": "GPUs with XID errors", "description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 24 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 1, "color": "red" } ] } }, "overrides": [] } }, { "type": "stat", "id": 32, "targets": [ { "expr": "avg(gpu:power_throttle_fraction:rate5m)", "refId": "A" } ], "title": "Power throttling (cluster avg)", "description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 24 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.2, "color": "red" } ] } }, "overrides": [] } }, { "type": "stat", "id": 33, "targets": [ { "expr": "avg(gpu:thermal_throttle_fraction:rate5m)", "refId": "A" } ], "title": "Thermal throttling (cluster avg)", "description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 24 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 0.05, "color": "yellow" }, { "value": 0.2, "color": "red" } ] } }, "overrides": [] } }, { "type": "stat", "id": 34, "targets": [ { "expr": "max(DCGM_FI_DEV_GPU_TEMP)", "refId": "A" } ], "title": "Max GPU temperature", "description": "Hottest GPU in the fleet right now. Adjust thresholds to match your GPU model specifications.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 24 }, "repeatDirection": "h", "options": { "graphMode": "none", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, "percentChangeColorMode": "standard", "orientation": "" }, "fieldConfig": { "defaults": { "unit": "celsius", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 75, "color": "yellow" }, { "value": 85, "color": "red" } ] } }, "overrides": [] } }, { "type": "timeseries", "id": 35, "targets": [ { "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "{{Hostname}}/{{gpu}}", "refId": "A" } ], "title": "Temperature per GPU", "description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "celsius", "min": 20, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 75, "color": "yellow" }, { "value": 85, "color": "red" } ] }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] } }, { "type": "timeseries", "id": 36, "targets": [ { "expr": "gpu:power_throttle_fraction:rate5m", "legendFormat": "power {{Hostname}}/{{gpu}}", "refId": "A" }, { "expr": "gpu:thermal_throttle_fraction:rate5m", "legendFormat": "thermal {{Hostname}}/{{gpu}}", "refId": "B" } ], "title": "Throttle fraction per GPU", "description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "repeatDirection": "h", "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": false, "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 25, "showPoints": "never" } }, "overrides": [] } }, { "type": "row", "collapsed": false, "title": "Hardware inventory", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 40, "panels": [] }, { "type": "table", "id": 41, "targets": [ { "expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)", "instant": true, "range": false, "format": "table", "refId": "A" } ], "title": "GPU inventory", "description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.", "transparent": false, "datasource": { "type": "prometheus", "uid": "$ds_prometheus" }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 38 }, "repeatDirection": "h", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "cluster": true, "container": true, "device": true, "endpoint": true, "instance": true, "job": true, "pci_bus_id": true, "prometheus": true, "service": true, "tenant": true, "tier": true, "uid": true, "unit": true }, "indexByName": { "Hostname": 0, "UUID": 3, "gpu": 2, "modelName": 1 }, "renameByName": { "Hostname": "Node", "UUID": "UUID", "gpu": "Index", "modelName": "GPU Model" } } } ], "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [] }, "cellHeight": "sm" } } ], "templating": { "list": [ { "type": "datasource", "name": "ds_prometheus", "label": "Prometheus", "skipUrlSync": false, "query": "prometheus", "current": { "selected": false, "text": "default", "value": "default" }, "multi": false, "allowCustomValue": true, "includeAll": false, "regex": "", "auto": false, "auto_min": "10s", "auto_count": 30 } ] }, "annotations": {} }