cozystack/hack/dcgm-default-counters.csv

1	# Snapshot of the upstream DCGM Exporter default-counters.csv used as a
2	# fixture by hack/check-gpu-recording-rules.bats. The test verifies that
3	# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either
4	# declared here (upstream defaults) or in
5	# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
6	# (the project's custom CSV).
7	#
8	# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv
9	# Pinned to the DCGM Exporter image tag shipped by the gpu-operator
10	# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml
11	# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is
12	# bumped	refresh this file from the matching tag in the NVIDIA/dcgm-exporter
13	# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`.
14	# Format
15	# If line starts with a '#' it is considered a comment
16	# DCGM FIELD	Prometheus metric type	help message
17	# Clocks
18	DCGM_FI_DEV_SM_CLOCK	gauge	SM clock frequency (in MHz).
19	DCGM_FI_DEV_MEM_CLOCK	gauge	Memory clock frequency (in MHz).
20	# Temperature
21	DCGM_FI_DEV_MEMORY_TEMP	gauge	Memory temperature (in C).
22	DCGM_FI_DEV_GPU_TEMP	gauge	GPU temperature (in C).
23	# Power
24	DCGM_FI_DEV_POWER_USAGE	gauge	Power draw (in W).
25	DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION	counter	Total energy consumption since boot (in mJ).
26	# PCIE
27	# DCGM_FI_PROF_PCIE_TX_BYTES	counter	Total number of bytes transmitted through PCIe TX via NVML.
28	# DCGM_FI_PROF_PCIE_RX_BYTES	counter	Total number of bytes received through PCIe RX via NVML.
29	DCGM_FI_DEV_PCIE_REPLAY_COUNTER	counter	Total number of PCIe retries.
30	# Utilization (the sample period varies depending on the product)
31	DCGM_FI_DEV_GPU_UTIL	gauge	GPU utilization (in %).
32	DCGM_FI_DEV_MEM_COPY_UTIL	gauge	Memory utilization (in %).
33	DCGM_FI_DEV_ENC_UTIL	gauge	Encoder utilization (in %).
34	DCGM_FI_DEV_DEC_UTIL	gauge	Decoder utilization (in %).
35	# Errors and violations
36	DCGM_FI_DEV_XID_ERRORS	gauge	Value of the last XID error encountered.
37	# DCGM_FI_DEV_POWER_VIOLATION	counter	Throttling duration due to power constraints (in us).
38	# DCGM_FI_DEV_THERMAL_VIOLATION	counter	Throttling duration due to thermal constraints (in us).
39	# DCGM_FI_DEV_SYNC_BOOST_VIOLATION	counter	Throttling duration due to sync-boost constraints (in us).
40	# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION	counter	Throttling duration due to board limit constraints (in us).
41	# DCGM_FI_DEV_LOW_UTIL_VIOLATION	counter	Throttling duration due to low utilization (in us).
42	# DCGM_FI_DEV_RELIABILITY_VIOLATION	counter	Throttling duration due to reliability constraints (in us).
43	# Memory usage
44	DCGM_FI_DEV_FB_FREE	gauge	Framebuffer memory free (in MiB).
45	DCGM_FI_DEV_FB_USED	gauge	Framebuffer memory used (in MiB).
46	# ECC
47	# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL	counter	Total number of single-bit volatile ECC errors.
48	# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL	counter	Total number of double-bit volatile ECC errors.
49	# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL	counter	Total number of single-bit persistent ECC errors.
50	# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL	counter	Total number of double-bit persistent ECC errors.
51	# Retired pages
52	# DCGM_FI_DEV_RETIRED_SBE	counter	Total number of retired pages due to single-bit errors.
53	# DCGM_FI_DEV_RETIRED_DBE	counter	Total number of retired pages due to double-bit errors.
54	# DCGM_FI_DEV_RETIRED_PENDING	counter	Total number of pages pending retirement.
55	# NVLink
56	# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL	counter	Total number of NVLink flow-control CRC errors.
57	# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL	counter	Total number of NVLink data CRC errors.
58	# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL	counter	Total number of NVLink retries.
59	# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL	counter	Total number of NVLink recovery errors.
60	DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL	counter	Total number of NVLink bandwidth counters for all lanes.
61	# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0	counter	The number of bytes of active NVLink rx or tx data including both header and payload.
62	# VGPU License status
63	DCGM_FI_DEV_VGPU_LICENSE_STATUS	gauge	vGPU License status
64	# Remapped rows
65	DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS	counter	Number of remapped rows for uncorrectable errors
66	DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS	counter	Number of remapped rows for correctable errors
67	DCGM_FI_DEV_ROW_REMAP_FAILURE	gauge	Whether remapping of rows has failed
68	# Static configuration information. These appear as labels on the other metrics
69	DCGM_FI_DRIVER_VERSION	label	Driver Version
70	# DCGM_FI_NVML_VERSION	label	NVML Version
71	# DCGM_FI_DEV_BRAND	label	Device Brand
72	# DCGM_FI_DEV_SERIAL	label	Device Serial Number
73	# DCGM_FI_DEV_OEM_INFOROM_VER	label	OEM inforom version
74	# DCGM_FI_DEV_ECC_INFOROM_VER	label	ECC inforom version
75	# DCGM_FI_DEV_POWER_INFOROM_VER	label	Power management object inforom version
76	# DCGM_FI_DEV_INFOROM_IMAGE_VER	label	Inforom image version
77	# DCGM_FI_DEV_VBIOS_VERSION	label	VBIOS version of the device
78	# Datacenter Profiling (DCP) metrics
79	# NOTE: supported on Nvidia datacenter Volta GPUs and newer
80	DCGM_FI_PROF_GR_ENGINE_ACTIVE	gauge	Ratio of time the graphics engine is active.
81	# DCGM_FI_PROF_SM_ACTIVE	gauge	The ratio of cycles an SM has at least 1 warp assigned.
82	# DCGM_FI_PROF_SM_OCCUPANCY	gauge	The ratio of number of warps resident on an SM.
83	DCGM_FI_PROF_PIPE_TENSOR_ACTIVE	gauge	Ratio of cycles the tensor (HMMA) pipe is active.
84	DCGM_FI_PROF_DRAM_ACTIVE	gauge	Ratio of cycles the device memory interface is active sending or receiving data.
85	# DCGM_FI_PROF_PIPE_FP64_ACTIVE	gauge	Ratio of cycles the fp64 pipes are active.
86	# DCGM_FI_PROF_PIPE_FP32_ACTIVE	gauge	Ratio of cycles the fp32 pipes are active.
87	# DCGM_FI_PROF_PIPE_FP16_ACTIVE	gauge	Ratio of cycles the fp16 pipes are active.
88	DCGM_FI_PROF_PCIE_TX_BYTES	gauge	The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
89	DCGM_FI_PROF_PCIE_RX_BYTES	gauge	The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.