Debian Codegraph @

src/gpu_nvidia.c

raw
1 /*
2 Copyright 2018 Evgeny Naumov
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy of
5 this software and associated documentation files (the "Software"), to deal in
6 the Software without restriction, including without limitation the rights to
7 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
8 of the Software, and to permit persons to whom the Software is furnished to do
9 so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 SOFTWARE.
21 */
22
23 #include "daemon/collectd.h"
24 #include "daemon/plugin.h"
25 #include "utils/common/common.h"
26
27 #include <nvml.h>
28 #include <stdint.h>
29 #include <stdio.h>
30
31 #define PLUGIN_NAME "gpu_nvidia"
32
33 static nvmlReturn_t nv_status = NVML_SUCCESS;
34 static char *nv_errline = "";
35
36 #define TRY_CATCH(f, catch) \
37 if ((nv_status = f) != NVML_SUCCESS) { \
38 nv_errline = #f; \
39 goto catch; \
40 }
41
42 #define TRY_CATCH_OPTIONAL(f, catch) \
43 if ((nv_status = f) != NVML_SUCCESS && \
44 nv_status != NVML_ERROR_NOT_SUPPORTED) { \
45 nv_errline = #f; \
46 goto catch; \
47 }
48
49 #define TRY(f) TRY_CATCH(f, catch)
50 #define TRYOPT(f) TRY_CATCH_OPTIONAL(f, catch)
51
52 #define KEY_GPUINDEX "GPUIndex"
53 #define KEY_IGNORESELECTED "IgnoreSelected"
54 #define KEY_INSTANCE_BY_GPUINDEX "InstanceByGPUIndex"
55 #define KEY_INSTANCE_BY_GPUNAME "InstanceByGPUName"
56
57 static const char *config_keys[] = {KEY_GPUINDEX, KEY_IGNORESELECTED,
58 KEY_INSTANCE_BY_GPUINDEX,
59 KEY_INSTANCE_BY_GPUNAME};
60 static const unsigned int n_config_keys = STATIC_ARRAY_SIZE(config_keys);
61
62 // This is a bitflag, necessitating the (extremely conservative) assumption
63 // that there are no more than 64 GPUs on this system.
64 static uint64_t conf_match_mask = 0;
65 static bool conf_mask_is_exclude = 0;
66
67 // use GPU index and device name by default
68 #define INSTANCE_BY_GPUINDEX (1 << 0)
69 #define INSTANCE_BY_GPUNAME (1 << 1)
70 static uint8_t instance_by = INSTANCE_BY_GPUINDEX | INSTANCE_BY_GPUNAME;
71
72 static int nvml_config(const char *key, const char *value) {
73 if (strcasecmp(key, KEY_GPUINDEX) == 0) {
74 char *eptr;
75 unsigned long device_ix = strtoul(value, &eptr, 10);
76 if (eptr == value) {
77 ERROR(PLUGIN_NAME ": Failed to parse GPUIndex value \"%s\"", value);
78 return -1;
79 }
80 if (device_ix >= 64) {
81 ERROR(PLUGIN_NAME
82 ": At most 64 GPUs (0 <= GPUIndex < 64) are supported!");
83 return -2;
84 }
85 conf_match_mask |= (1 << device_ix);
86 } else if (strcasecmp(key, KEY_IGNORESELECTED) == 0) {
87 conf_mask_is_exclude = IS_TRUE(value);
88 } else if (strcasecmp(key, KEY_INSTANCE_BY_GPUINDEX) == 0) {
89 // if KEY_INSTANCE_BY_GPUINDEX is set to false, set bit to 0
90 if (IS_FALSE(value)) {
91 instance_by &= ~INSTANCE_BY_GPUINDEX;
92 }
93 } else if (strcasecmp(key, KEY_INSTANCE_BY_GPUNAME) == 0) {
94 // if KEY_INSTANCE_BY_GPUNAME is set to false, set bit to 0
95 if (IS_FALSE(value)) {
96 instance_by &= ~INSTANCE_BY_GPUNAME;
97 }
98 } else {
99 ERROR(PLUGIN_NAME ": Unrecognized config option %s", key);
100 return -10;
101 }
102
103 return 0;
104 }
105
106 static int nvml_init(void) {
107 TRY(nvmlInit());
108 return 0;
109
110 catch : ERROR(PLUGIN_NAME ": NVML init failed with %d", nv_status);
111 return -1;
112 }
113
114 static int nvml_shutdown(void) {
115 TRY(nvmlShutdown())
116 return 0;
117
118 catch : ERROR(PLUGIN_NAME ": NVML shutdown failed with %d", nv_status);
119 return -1;
120 }
121
122 static void nvml_submit_gauge(int device_idx, const char *device_name,
123 const char *type, const char *type_instance,
124 gauge_t nvml) {
125
126 value_list_t vl = VALUE_LIST_INIT;
127
128 vl.values = &(value_t){.gauge = nvml};
129 vl.values_len = 1;
130
131 sstrncpy(vl.plugin, PLUGIN_NAME, sizeof(vl.plugin));
132
133 // use gpu index and name or either of the two
134 if (instance_by == (INSTANCE_BY_GPUNAME | INSTANCE_BY_GPUINDEX)) {
135 snprintf(vl.plugin_instance, sizeof(vl.plugin_instance), "%i-%s",
136 device_idx, device_name);
137 } else if (instance_by & INSTANCE_BY_GPUINDEX) {
138 snprintf(vl.plugin_instance, sizeof(vl.plugin_instance), "%i", device_idx);
139 } else if (instance_by & INSTANCE_BY_GPUNAME) {
140 sstrncpy(vl.plugin_instance, device_name, sizeof(vl.plugin_instance));
141 }
142
143 sstrncpy(vl.type, type, sizeof(vl.type));
144
145 if (type_instance != NULL) {
146 sstrncpy(vl.type_instance, type_instance, sizeof(vl.type_instance));
147 }
148
149 plugin_dispatch_values(&vl);
150 }
151
152 static int nvml_read(void) {
153
154 unsigned int device_count;
155 TRY_CATCH(nvmlDeviceGetCount(&device_count), catch_nocount);
156
157 if (device_count > 64) {
158 device_count = 64;
159 }
160
161 for (unsigned int ix = 0; ix < device_count; ix++) {
162
163 unsigned int is_match =
164 ((1 << ix) & conf_match_mask) || (conf_match_mask == 0);
165 if (conf_mask_is_exclude == !!is_match) {
166 continue;
167 }
168
169 nvmlDevice_t dev;
170 TRY(nvmlDeviceGetHandleByIndex(ix, &dev));
171
172 char dev_name[NVML_DEVICE_NAME_BUFFER_SIZE] = {0};
173 if (instance_by & INSTANCE_BY_GPUNAME) {
174 TRY(nvmlDeviceGetName(dev, dev_name, NVML_DEVICE_NAME_BUFFER_SIZE));
175 }
176
177 // Try to be as lenient as possible with the variety of devices that are
178 // out there, ignoring any NOT_SUPPORTED errors gently.
179 nvmlMemory_t meminfo;
180 TRYOPT(nvmlDeviceGetMemoryInfo(dev, &meminfo))
181 if (nv_status == NVML_SUCCESS) {
182 nvml_submit_gauge(ix, dev_name, "memory", "used", meminfo.used);
183 nvml_submit_gauge(ix, dev_name, "memory", "free", meminfo.free);
184 }
185
186 nvmlUtilization_t utilization;
187 TRYOPT(nvmlDeviceGetUtilizationRates(dev, &utilization))
188 if (nv_status == NVML_SUCCESS)
189 nvml_submit_gauge(ix, dev_name, "percent", "gpu_used", utilization.gpu);
190
191 unsigned int fan_speed;
192 TRYOPT(nvmlDeviceGetFanSpeed(dev, &fan_speed))
193 if (nv_status == NVML_SUCCESS)
194 nvml_submit_gauge(ix, dev_name, "fanspeed", NULL, fan_speed);
195
196 unsigned int core_temp;
197 TRYOPT(nvmlDeviceGetTemperature(dev, NVML_TEMPERATURE_GPU, &core_temp))
198 if (nv_status == NVML_SUCCESS)
199 nvml_submit_gauge(ix, dev_name, "temperature", "core", core_temp);
200
201 unsigned int sm_clk_mhz;
202 TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_SM, &sm_clk_mhz))
203 if (nv_status == NVML_SUCCESS)
204 nvml_submit_gauge(ix, dev_name, "frequency", "multiprocessor",
205 1e6 * sm_clk_mhz);
206
207 unsigned int mem_clk_mhz;
208 TRYOPT(nvmlDeviceGetClockInfo(dev, NVML_CLOCK_MEM, &mem_clk_mhz))
209 if (nv_status == NVML_SUCCESS)
210 nvml_submit_gauge(ix, dev_name, "frequency", "memory", 1e6 * mem_clk_mhz);
211
212 unsigned int power_mW;
213 TRYOPT(nvmlDeviceGetPowerUsage(dev, &power_mW))
214 if (nv_status == NVML_SUCCESS)
215 nvml_submit_gauge(ix, dev_name, "power", NULL, 1e-3 * power_mW);
216
217 continue;
218
219 // Failures here indicate transient errors or removal of GPU. In either
220 // case it will either be resolved or the GPU will no longer be enumerated
221 // the next time round.
222 catch : WARNING(PLUGIN_NAME
223 ": NVML call \"%s\" failed (%d) on dev at index %d!",
224 nv_errline, nv_status, ix);
225 continue;
226 }
227
228 return 0;
229
230 // Failures here indicate serious misconfiguration; we bail out totally.
231 catch_nocount:
232 ERROR(PLUGIN_NAME ": Failed to enumerate NVIDIA GPUs (\"%s\" returned %d)",
233 nv_errline, nv_status);
234 return -1;
235 }
236
237 void module_register(void) {
238 plugin_register_init(PLUGIN_NAME, nvml_init);
239 plugin_register_config(PLUGIN_NAME, nvml_config, config_keys, n_config_keys);
240 plugin_register_read(PLUGIN_NAME, nvml_read);
241 plugin_register_shutdown(PLUGIN_NAME, nvml_shutdown);
242 }