{
  "uid": "gpu-tenants",
  "title": "GPU Tenant Usage",
  "description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.",
  "tags": [
    "gpu",
    "tenants",
    "admin"
  ],
  "timezone": "browser",
  "editable": true,
  "graphTooltip": 1,
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "fiscalYearStartMonth": 0,
  "schemaVersion": 42,
  "panels": [
    {
      "type": "row",
      "collapsed": false,
      "title": "Tenant fleet snapshot",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "panels": []
    },
    {
      "type": "stat",
      "id": 2,
      "targets": [
        {
          "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)",
          "refId": "A"
        }
      ],
      "title": "Active tenants",
      "description": "Namespaces with at least one GPU request (Pending + Running).",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 0,
        "y": 1
      },
      "repeatDirection": "h",
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "justifyMode": "auto",
        "textMode": "value",
        "wideLayout": true,
        "showPercentChange": false,
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "percentChangeColorMode": "standard",
        "orientation": ""
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "id": 3,
      "targets": [
        {
          "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})",
          "refId": "A"
        }
      ],
      "title": "Allocated GPUs",
      "description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 4,
        "y": 1
      },
      "repeatDirection": "h",
      "options": {
        "graphMode": "none",
        "colorMode": "background",
        "justifyMode": "auto",
        "textMode": "value",
        "wideLayout": true,
        "showPercentChange": false,
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "percentChangeColorMode": "standard",
        "orientation": ""
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "blue"
              },
              {
                "value": 1,
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "id": 4,
      "targets": [
        {
          "expr": "cluster:gpu_util:avg",
          "refId": "A"
        }
      ],
      "title": "Avg cluster GPU util",
      "description": "Average NVML utilization across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 8,
        "y": 1
      },
      "repeatDirection": "h",
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "justifyMode": "auto",
        "textMode": "value",
        "wideLayout": true,
        "showPercentChange": false,
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "percentChangeColorMode": "standard",
        "orientation": ""
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "max": 100,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "blue"
              },
              {
                "value": 30,
                "color": "green"
              },
              {
                "value": 80,
                "color": "orange"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "id": 5,
      "targets": [
        {
          "expr": "cluster:gpu_power_watts:sum",
          "refId": "A"
        }
      ],
      "title": "Total GPU power",
      "description": "Live power draw summed across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 12,
        "y": 1
      },
      "repeatDirection": "h",
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "justifyMode": "auto",
        "textMode": "value",
        "wideLayout": true,
        "showPercentChange": false,
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "percentChangeColorMode": "standard",
        "orientation": ""
      },
      "fieldConfig": {
        "defaults": {
          "unit": "watt",
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "id": 6,
      "targets": [
        {
          "expr": "sum(sum_over_time(node:power_watts:sum[24h:1m])) / 60 / 1000",
          "refId": "A"
        }
      ],
      "title": "Energy (last 24h)",
      "description": "Integrated cluster-wide GPU power draw over 24h. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 16,
        "y": 1
      },
      "repeatDirection": "h",
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "justifyMode": "auto",
        "textMode": "value",
        "wideLayout": true,
        "showPercentChange": false,
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "percentChangeColorMode": "standard",
        "orientation": ""
      },
      "fieldConfig": {
        "defaults": {
          "unit": "kwatth",
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "id": 7,
      "targets": [
        {
          "expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60",
          "refId": "A"
        }
      ],
      "title": "GPU-hours (last 24h)",
      "description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 20,
        "y": 1
      },
      "repeatDirection": "h",
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "justifyMode": "auto",
        "textMode": "value",
        "wideLayout": true,
        "showPercentChange": false,
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "percentChangeColorMode": "standard",
        "orientation": ""
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "collapsed": false,
      "title": "Current allocation",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 5
      },
      "id": 10,
      "panels": []
    },
    {
      "type": "bargauge",
      "id": 11,
      "targets": [
        {
          "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
          "legendFormat": "{{namespace}}",
          "refId": "A"
        }
      ],
      "title": "GPUs per namespace",
      "description": "Kube GPU requests per tenant (billable). Includes Pending pods.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 6
      },
      "repeatDirection": "h",
      "options": {
        "displayMode": "gradient",
        "valueMode": "color",
        "namePlacement": "auto",
        "showUnfilled": true,
        "sizing": "auto",
        "minVizWidth": 8,
        "minVizHeight": 16,
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false,
          "calcs": []
        },
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "maxVizHeight": 300,
        "orientation": "horizontal"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "bargauge",
      "id": 12,
      "targets": [
        {
          "expr": "node:fb_used_bytes:sum",
          "legendFormat": "{{Hostname}}",
          "refId": "A"
        }
      ],
      "title": "VRAM used per node",
      "description": "FB memory actively used per node. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 6
      },
      "repeatDirection": "h",
      "options": {
        "displayMode": "gradient",
        "valueMode": "color",
        "namePlacement": "auto",
        "showUnfilled": true,
        "sizing": "auto",
        "minVizWidth": 8,
        "minVizHeight": 16,
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false,
          "calcs": []
        },
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "maxVizHeight": 300,
        "orientation": "horizontal"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "bytes",
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "collapsed": false,
      "title": "Utilization (per-node)",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 14
      },
      "id": 20,
      "panels": []
    },
    {
      "type": "timeseries",
      "id": 21,
      "targets": [
        {
          "expr": "node:gpu_util:avg",
          "legendFormat": "{{Hostname}}",
          "refId": "A"
        }
      ],
      "title": "NVML utilization per node",
      "description": "Per-node NVML utilization. Stacked to show fleet-wide pressure over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 15
      },
      "repeatDirection": "h",
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": false,
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": ""
        }
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "custom": {
            "drawStyle": "line",
            "lineInterpolation": "linear",
            "fillOpacity": 40,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "id": 22,
      "targets": [
        {
          "expr": "node:tensor_active:avg * 100",
          "legendFormat": "{{Hostname}}",
          "refId": "A"
        }
      ],
      "title": "Tensor saturation per node",
      "description": "Real tensor core load per node. Useful to spot nodes with idle tensor cores. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 15
      },
      "repeatDirection": "h",
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": false,
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": ""
        }
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "min": 0,
          "custom": {
            "drawStyle": "line",
            "lineInterpolation": "linear",
            "fillOpacity": 40,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "collapsed": false,
      "title": "Power \u0026 allocation trend",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 30,
      "panels": []
    },
    {
      "type": "timeseries",
      "id": 31,
      "targets": [
        {
          "expr": "node:power_watts:sum",
          "legendFormat": "{{Hostname}}",
          "refId": "A"
        }
      ],
      "title": "Power draw per node",
      "description": "Per-node GPU power draw. Stacked to see cluster-wide energy burn over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 24
      },
      "repeatDirection": "h",
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": false,
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": ""
        }
      },
      "fieldConfig": {
        "defaults": {
          "unit": "watt",
          "min": 0,
          "custom": {
            "drawStyle": "line",
            "lineInterpolation": "linear",
            "fillOpacity": 40,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "id": 32,
      "targets": [
        {
          "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
          "legendFormat": "{{namespace}}",
          "refId": "A"
        }
      ],
      "title": "Allocated GPUs per namespace",
      "description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 24
      },
      "repeatDirection": "h",
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "showLegend": false,
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": ""
        }
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "min": 0,
          "custom": {
            "drawStyle": "line",
            "lineInterpolation": "stepAfter",
            "fillOpacity": 40,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "collapsed": false,
      "title": "Top consumers (live)",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 32
      },
      "id": 40,
      "panels": []
    },
    {
      "type": "table",
      "id": 41,
      "targets": [
        {
          "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}",
          "instant": true,
          "range": false,
          "format": "table",
          "refId": "gpus"
        }
      ],
      "title": "Top consumers (allocation)",
      "description": "Live ranking of tenants by GPU allocation. Hardware metrics (util, tensor, power, VRAM) cannot be attributed to workload namespaces — see Fleet and Performance dashboards for per-node/per-GPU hardware views.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 33
      },
      "repeatDirection": "h",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "cluster": true,
              "cluster 1": true,
              "cluster 2": true,
              "cluster 3": true,
              "cluster 4": true,
              "cluster 5": true,
              "endpoint": true,
              "endpoint 1": true,
              "endpoint 2": true,
              "endpoint 3": true,
              "endpoint 4": true,
              "endpoint 5": true,
              "instance": true,
              "instance 1": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "job": true,
              "job 1": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "prometheus": true,
              "prometheus 1": true,
              "prometheus 2": true,
              "prometheus 3": true,
              "prometheus 4": true,
              "prometheus 5": true,
              "service": true,
              "service 1": true,
              "service 2": true,
              "service 3": true,
              "service 4": true,
              "service 5": true,
              "tenant": true,
              "tenant 1": true,
              "tenant 2": true,
              "tenant 3": true,
              "tenant 4": true,
              "tenant 5": true,
              "tier": true,
              "tier 1": true,
              "tier 2": true,
              "tier 3": true,
              "tier 4": true,
              "tier 5": true,
              "uid": true,
              "uid 1": true,
              "uid 2": true,
              "uid 3": true,
              "uid 4": true,
              "uid 5": true,
              "unit": true,
              "unit 1": true,
              "unit 2": true,
              "unit 3": true,
              "unit 4": true,
              "unit 5": true
            },
            "indexByName": {
              "Value #gpus": 1,
              "namespace": 0
            },
            "renameByName": {
              "Value #gpus": "GPUs",
              "namespace": "Namespace"
            }
          }
        }
      ],
      "options": {
        "frameIndex": 0,
        "showHeader": true,
        "showTypeIcons": false,
        "sortBy": [
          {
            "displayName": "GPUs",
            "desc": true
          }
        ],
        "footer": {
          "show": false,
          "reducer": []
        },
        "cellHeight": "sm"
      }
    },
    {
      "type": "row",
      "collapsed": false,
      "title": "Historical usage (last 24h)",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 43
      },
      "id": 50,
      "panels": []
    },
    {
      "type": "bargauge",
      "id": 51,
      "targets": [
        {
          "expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60",
          "legendFormat": "{{namespace}}",
          "refId": "A"
        }
      ],
      "title": "GPU-hours per namespace (last 24h)",
      "description": "Billable GPU-hours per tenant over the last 24h window.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 44
      },
      "repeatDirection": "h",
      "options": {
        "displayMode": "gradient",
        "valueMode": "color",
        "namePlacement": "auto",
        "showUnfilled": true,
        "sizing": "auto",
        "minVizWidth": 8,
        "minVizHeight": 16,
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false,
          "calcs": []
        },
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "maxVizHeight": 300,
        "orientation": "horizontal"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "blue"
              }
            ]
          }
        },
        "overrides": []
      }
    },
    {
      "type": "bargauge",
      "id": 52,
      "targets": [
        {
          "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000",
          "legendFormat": "{{Hostname}}",
          "refId": "A"
        }
      ],
      "title": "Energy per node (last 24h)",
      "description": "kWh consumed per node over the last 24h. Integrated from 1-minute power samples. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.",
      "transparent": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 44
      },
      "repeatDirection": "h",
      "options": {
        "displayMode": "gradient",
        "valueMode": "color",
        "namePlacement": "auto",
        "showUnfilled": true,
        "sizing": "auto",
        "minVizWidth": 8,
        "minVizHeight": 16,
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false,
          "calcs": []
        },
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "maxVizHeight": 300,
        "orientation": "horizontal"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "kwatth",
          "decimals": 2,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "value": null,
                "color": "green"
              }
            ]
          }
        },
        "overrides": []
      }
    }
  ],
  "templating": {
    "list": [
      {
        "type": "datasource",
        "name": "ds_prometheus",
        "label": "Prometheus",
        "skipUrlSync": false,
        "query": "prometheus",
        "current": {
          "selected": false,
          "text": "default",
          "value": "default"
        },
        "multi": false,
        "allowCustomValue": true,
        "includeAll": false,
        "regex": "",
        "auto": false,
        "auto_min": "10s",
        "auto_count": 30
      },
      {
        "type": "query",
        "name": "namespace",
        "label": "Namespace",
        "skipUrlSync": false,
        "query": "label_values(namespace:gpu_count:allocated, namespace)",
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "multi": true,
        "allowCustomValue": true,
        "refresh": 2,
        "sort": 1,
        "includeAll": true,
        "auto": false,
        "auto_min": "10s",
        "auto_count": 30
      }
    ]
  },
  "annotations": {}
}