Debian Codegraph @

gmond/python_modules/disk/diskstat.py

raw
1 #!/usr/bin/env python
2 # This script reports disk stat metrics to ganglia.
3 #
4 # Notes:
5 # This script exposes values in /proc/diskstats and calculates
6 # various statistics based on the Linux kernel 2.6. To find
7 # more information on these values, look in the Linux kernel
8 # documentation for "I/O statistics fields".
9 #
10 # By default, the script would monitor any entry listed under
11 # /proc/diskstats that is not containing a number. Override it by passing
12 # a list of devices in the 'devices' parameter.
13 #
14 # This script has the option of explicitly setting which devices
15 # to check using the "devices" option in your configuration. If
16 # you set this value, it will invalidate the MIN_DISK_SIZE and
17 # IGNORE_DEV options described below. This enables you to
18 # monitor specific partitions instead of the entire device.
19 # Example value: "sda1 sda2".
20 # Example value: "sda sdb sdc".
21 #
22 # This script also checks for a minimum disk size in order to
23 # only measure interesting devices by default.
24 # [Can be overriden if "devices" is set]
25 #
26 # This script looks for disks to check in /proc/partitions while
27 # ignoring any devices present in IGNORE_DEV by default.
28 # [Can be overriden if "devices" is set]
29 #
30 # Example Metrics
31 # * diskstat_XXX_io_time
32 # * diskstat_XXX_read_kbytes_per_sec
33 # * diskstat_XXX_read_time
34 # * diskstat_XXX_reads
35 # * diskstat_XXX_reads_merged
36 # * diskstat_XXX_weighted_io_time
37 # * diskstat_XXX_write_kbytes_per_sec
38 # * diskstat_XXX_write_time
39 # * diskstat_XXX_writes
40 # * diskstat_XXX_writes_merged
41 # Changelog:
42 # v1.0.1 - 2010-07-22
43 # * Initial version
44 #
45 # v1.0.2 - 2010-08-03
46 # * Modified reads_per_sec to not calculate per second delta.
47 # This enables us to generate a better graph by stacking
48 # reads/writes with reads/writes merged.
49 #
50
51 # Copyright Jamie Isaacs. 2010
52 # License to use, modify, and distribute under the GPL
53 # http://www.gnu.org/licenses/gpl.txt
54
55 import logging
56 import os
57 import re
58 import stat
59 import time
60
61 descriptors = []
62
63 logging.basicConfig(level=logging.ERROR, format="%(asctime)s - %(name)s - %(levelname)s\t Thread-%(thread)d - %(message)s")
64 logging.debug('starting up')
65
66 last_update = 0
67 cur_time = 0
68 stats = {}
69 last_val = {}
70
71 MAX_UPDATE_TIME = 15
72 BYTES_PER_SECTOR = 512
73
74 # 5 GB
75 MIN_DISK_SIZE = 5242880
76 # Set to None to trigger disk discovery under /proc/diskstats
77 # Pass a 'devices' parameter to explicitly list disks to monitor
78 DEVICES = None
79 IGNORE_DEV = 'dm-|loop|drbd'
80
81 PARTITIONS_FILE = '/proc/partitions'
82 DISKSTATS_FILE = '/proc/diskstats'
83 DMDIR = '/dev/mapper'
84 device_mapper = ''
85
86 PARTITIONS = []
87 dmnames = []
88 dm2pair = {}
89 pair2dm = {}
90 devnames = []
91 dev2pair = {}
92 pair2dev = {}
93
94
95 def build_dmblock_major_minor_tables():
96 """Returns
97 1) a table of filenames that are all device mapper block special files
98 2) a dict mapping each device mapper name to (major,minor)
99 3) a dict mapping each (major,minor) pair to a table of devce mapper names"""
100
101 names = []
102 name2pair = {}
103 pair2name = {}
104 mapper_entries = []
105
106 mapper_entries = os.listdir(DMDIR)
107 for n in mapper_entries:
108 s = os.lstat(DMDIR + '/' + n)
109 if stat.S_ISBLK(s[stat.ST_MODE]):
110 names.append(n)
111 maj = str(os.major(s.st_rdev))
112 min = str(os.minor(s.st_rdev))
113 name2pair[n] = (maj, min)
114 pair2name[(maj, min)] = n
115
116 logging.debug('grabbed dmsetup device info')
117 logging.debug('dmsetup devices: ' + str(name2pair))
118
119 return (names, name2pair, pair2name)
120
121
122 def build_block_major_minor_tables():
123 """Returns
124 1) a table of filenames that are all block special files
125 2) a dict mapping each dev name to (major,minor)
126 3) a dict mapping each (major,minor) pair to a table of dev names"""
127 dnames = []
128 d2p = {}
129 p2d = {}
130
131 # Get values from diskstats file
132 lines = tuple(open(PARTITIONS_FILE, 'r'))
133 logging.debug('grabbed diskstat device info')
134 logging.debug('diskstat devices: ' + '\n'.join(lines))
135
136 for line in lines:
137 if line:
138 # read the first three fields from each line
139 (maj, min, name) = line.split()[:3]
140 dnames.append(name)
141 d2p[name] = (maj, min)
142 p2d[(maj, min)] = name
143
144 return (dnames, d2p, p2d)
145
146
147 def get_devname(dev):
148 """Returns device mapper name converted to dev name"""
149
150 (maj, min) = dm2pair[dev]
151 name = pair2dev[(maj, min)]
152 return name
153
154
155 def list_dmnames():
156 """Returns string of device names associated with device mapper names"""
157
158 global dmnames
159 global dm2pair
160 global pair2dm
161 global devnames
162 global dev2pair
163 global pair2dev
164 devlist = ''
165
166 dmnames, dm2pair, pair2dm = build_dmblock_major_minor_tables()
167 logging.debug('dmnames: ' + str(dmnames))
168
169 devnames, dev2pair, pair2dev = build_block_major_minor_tables()
170 logging.debug('devnames: ' + str(dmnames))
171
172 for d in dmnames:
173 devlist = devlist + ' ' + str(d)
174
175 logging.debug('devlist: ' + str(devlist))
176
177 return devlist
178
179
180 def get_partitions():
181 logging.debug('getting partitions')
182 global PARTITIONS
183
184 # We need DEVICES != "" because on at least Centos 6 this evaluates to true
185 if DEVICES is not None and DEVICES != "":
186 # Explicit device list has been set
187 logging.debug(' DEVICES has already been set')
188 out = DEVICES
189
190 else:
191 # Load partitions
192 devices = []
193 lines = tuple(open(PARTITIONS_FILE, 'r'))
194 for line in lines:
195 # There needs to be a major value or otherwise we skip that line. Skips headers etc.
196 if not re.search("(\w+)?\d+", line):
197 continue
198 device_name = line.split()[3]
199 device_ends_with_number = re.search('\d$', device_name)
200 if 'md' in device_name or not device_ends_with_number:
201 # only include md devices and base block devices
202 devices.append(device_name)
203 out = ' '.join(devices)
204 logging.debug(' result: ' + out)
205
206 for dev in out.split():
207 if DEVICES is not None:
208 # Explicit device list has been set
209 PARTITIONS.append(dev)
210 else:
211 # Load disk block size
212 c = open('/sys/block/' + dev + '/size', 'r').read(256)
213
214 # Make sure device is large enough to collect stats
215 if (int(c) * BYTES_PER_SECTOR / 1024) > MIN_DISK_SIZE:
216 PARTITIONS.append(dev)
217 else:
218 logging.debug(' ignoring ' + dev + ' due to size constraints')
219
220 return 0
221
222
223 ###########################################################################
224 # This is the order of metrics in /proc/diskstats
225 # 0 major Major number
226 # 1 minor Minor number
227 # 2 blocks Blocks
228 # 3 name Name
229 # 4 reads This is the total number of reads completed successfully.
230 # 5 merge_read Reads and writes which are adjacent to each other may be merged for
231 # efficiency. Thus two 4K reads may become one 8K read before it is
232 # ultimately handed to the disk, and so it will be counted (and queued)
233 # as only one I/O. This field lets you know how often this was done.
234 # 6 s_read This is the total number of sectors read successfully.
235 # 7 ms_read This is the total number of milliseconds spent by all reads.
236 # 8 writes This is the total number of writes completed successfully.
237 # 9 merge_write Reads and writes which are adjacent to each other may be merged for
238 # efficiency. Thus two 4K reads may become one 8K read before it is
239 # ultimately handed to the disk, and so it will be counted (and queued)
240 # as only one I/O. This field lets you know how often this was done.
241 # 10 s_write This is the total number of sectors written successfully.
242 # 11 ms_write This is the total number of milliseconds spent by all writes.
243 # 12 ios The only field that should go to zero. Incremented as requests are
244 # given to appropriate request_queue_t and decremented as they finish.
245 # 13 ms_io This field is increases so long as field 9 is nonzero.
246 # 14 ms_weighted This field is incremented at each I/O start, I/O completion, I/O
247 ###########################################################################
248
249 def update_stats():
250 logging.debug('updating stats')
251 global last_update, stats, last_val, cur_time
252 global MAX_UPDATE_TIME
253
254 cur_time = time.time()
255
256 if cur_time - last_update < MAX_UPDATE_TIME:
257 logging.debug(' wait ' + str(int(MAX_UPDATE_TIME - (cur_time - last_update))) + ' seconds')
258 return True
259
260 # Update diskstats
261 stats = {}
262
263 if not PARTITIONS:
264 part = get_partitions()
265 if part:
266 # Fail if return is non-zero
267 logging.warning('error getting partitions')
268 return False
269
270 # Get values for each disk device
271 for dev in PARTITIONS:
272 dev = dev.replace('/', '-')
273 logging.debug(" dev: " + dev)
274
275 # Setup storage lists
276 if dev not in stats:
277 stats[dev] = {}
278 if dev not in last_val:
279 last_val[dev] = {}
280
281 # Convert from dmname to devname
282 if device_mapper == 'true':
283 olddev = dev
284 dev = get_devname(dev)
285
286 # Get values from diskstats file
287 lines = tuple(open(DISKSTATS_FILE, 'r'))
288 for line in lines:
289 if dev in line.split():
290 vals = line.split()
291
292 logging.debug(' vals: ' + str(vals))
293
294 # Reset back to orignal dev name
295 if device_mapper == 'true':
296 dev = olddev
297
298 get_diff(dev, 'reads', int(vals[3]))
299 get_diff(dev, 'writes', int(vals[7]))
300
301 get_diff(dev, 'reads_merged', int(vals[4]))
302 get_diff(dev, 'writes_merged', int(vals[8]))
303
304 get_delta(dev, 'read_bytes_per_sec', int(vals[5]), float(BYTES_PER_SECTOR))
305 get_delta(dev, 'write_bytes_per_sec', int(vals[9]), float(BYTES_PER_SECTOR))
306
307 get_delta(dev, 'read_time', float(vals[6]), 0.001)
308 get_delta(dev, 'write_time', float(vals[10]), 0.001)
309
310 get_diff(dev, 'io_time', float(vals[12]), 0.001)
311 get_percent_time(dev, 'percent_io_time', float(stats[dev]['io_time']))
312 get_delta(dev, 'weighted_io_time', float(vals[13]), 0.001)
313
314 logging.debug('success refreshing stats')
315 logging.debug('stats: ' + str(stats))
316 logging.debug('last_val: ' + str(last_val))
317
318 last_update = cur_time
319 return True
320
321
322 def get_delta(dev, key, val, convert=1):
323 logging.debug(' get_delta for ' + dev + '_' + key)
324 global stats, last_val
325
326 if convert == 0:
327 logging.warning(' convert is zero!')
328
329 interval = cur_time - last_update
330
331 if key in last_val[dev] and interval > 0:
332
333 if val < last_val[dev][key]:
334 logging.debug(' fixing int32 wrap')
335 val += 4294967296
336
337 stats[dev][key] = (val - last_val[dev][key]) * float(convert) / float(interval)
338 else:
339 stats[dev][key] = 0
340
341 last_val[dev][key] = int(val)
342
343
344 def get_percent_time(dev, key, val):
345 logging.debug(' get_percent_time for ' + dev + '_' + key)
346 global stats, last_val
347
348 interval = cur_time - last_update
349
350 if interval > 0:
351 stats[dev][key] = (val / interval) * 100
352 else:
353 stats[dev][key] = 0
354
355
356 def get_diff(dev, key, val, convert=1):
357 logging.debug(' get_diff for ' + dev + '_' + key)
358 global stats, last_val
359
360 if key in last_val[dev]:
361 stats[dev][key] = (val - last_val[dev][key]) * float(convert)
362 else:
363 stats[dev][key] = 0
364
365 # If for some reason we have a negative diff we should assume counters reset
366 # and should set it back to 0
367 if stats[dev][key] < 0:
368 stats[dev][key] = 0
369
370 last_val[dev][key] = val
371
372
373 def get_stat(name):
374 logging.debug(' getting stat: ' + name)
375 global stats
376
377 ret = update_stats()
378
379 if ret:
380 if name.startswith('diskstat_'):
381 fir = name.find('_')
382 sec = name.find('_', fir + 1)
383
384 dev = name[fir + 1:sec]
385 label = name[sec + 1:]
386
387 try:
388 return stats[dev][label]
389 except:
390 logging.warning('failed to fetch [' + dev + '] ' + name)
391 return 0
392 else:
393 label = name
394
395 try:
396 return stats[label]
397 except:
398 logging.warning('failed to fetch ' + name)
399 return 0
400
401 else:
402 return 0
403
404
405 def metric_init(params):
406 global descriptors, device_mapper
407 global MIN_DISK_SIZE, DEVICES, IGNORE_DEV
408
409 # Use params.get here to assure function via gmond
410 if params.get('device-mapper') == 'true':
411 devices = list_dmnames()
412 DEVICES = devices
413 IGNORE_DEV = 'loop|drbd'
414 device_mapper = 'true'
415 logging.debug('dm block devices: ' + str(devices))
416 else:
417 DEVICES = params.get('devices')
418
419 logging.debug('init: ' + str(params))
420
421 time_max = 60
422
423 descriptions = dict(
424 reads={
425 'units': 'reads',
426 'description': 'The number of reads completed'},
427
428 reads_merged={
429 'units': 'reads',
430 'description': 'The number of reads merged. Reads which are adjacent to each other may be merged for efficiency. Multiple reads may become one before it is handed to the disk, and it will be counted (and queued) as only one I/O.'},
431
432 read_bytes_per_sec={
433 'units': 'bytes/sec',
434 'description': 'The number of bytes read per second'},
435
436 read_time={
437 'units': 's',
438 'description': 'The time in seconds spent reading'},
439
440 writes={
441 'units': 'writes',
442 'description': 'The number of writes completed'},
443
444 writes_merged={
445 'units': 'writes',
446 'description': 'The number of writes merged. Writes which are adjacent to each other may be merged for efficiency. Multiple writes may become one before it is handed to the disk, and it will be counted (and queued) as only one I/O.'},
447
448 write_bytes_per_sec={
449 'units': 'bytes/sec',
450 'description': 'The number of bytes written per second'},
451
452 write_time={
453 'units': 's',
454 'description': 'The time in seconds spent writing'},
455
456 io_time={
457 'units': 's',
458 'description': 'The time in seconds spent in I/O operations'},
459
460 percent_io_time={
461 'units': 'percent',
462 'value_type': 'float',
463 'format': '%f',
464 'description': 'The percent of disk time spent on I/O operations'},
465
466 weighted_io_time={
467 'units': 's',
468 'description': 'The weighted time in seconds spend in I/O operations. This measures each I/O start, I/O completion, I/O merge, or read of these stats by the number of I/O operations in progress times the number of seconds spent doing I/O.'}
469 )
470
471 update_stats()
472
473 for label in descriptions:
474 for dev in PARTITIONS:
475 if label in stats[dev]:
476
477 d = {
478 'name': 'diskstat_' + dev + '_' + label,
479 'call_back': get_stat,
480 'time_max': time_max,
481 'value_type': 'float',
482 'units': '',
483 'slope': 'both',
484 'format': '%f',
485 'description': label,
486 'groups': 'diskstat'
487 }
488
489 # Apply metric customizations from descriptions
490 d.update(descriptions[label])
491
492 descriptors.append(d)
493 else:
494 logging.error("skipped " + label)
495
496 # For command line testing
497 # time.sleep(MAX_UPDATE_TIME)
498 # update_stats()
499
500 return descriptors
501
502
503 def metric_cleanup():
504 logging.shutdown()
505 # pass
506
507 if __name__ == '__main__':
508 from optparse import OptionParser
509 import os
510
511 logging.debug('running from cmd line')
512 parser = OptionParser()
513 parser.add_option('-d', '--devices', dest='devices', default=None, help='devices to explicitly check')
514 parser.add_option('-b', '--gmetric-bin', dest='gmetric_bin', default='/usr/bin/gmetric', help='path to gmetric binary')
515 parser.add_option('-c', '--gmond-conf', dest='gmond_conf', default='/etc/ganglia/gmond.conf', help='path to gmond.conf')
516 parser.add_option('-g', '--gmetric', dest='gmetric', action='store_true', default=False, help='submit via gmetric')
517 parser.add_option('-m', '--device-mapper', dest='device_mapper', action='store_true', default=False, help='utilize all device mapper devices if set')
518 parser.add_option('-q', '--quiet', dest='quiet', action='store_true', default=False)
519
520 (options, args) = parser.parse_args()
521
522 if options.device_mapper:
523 metric_init({
524 'device-mapper': 'true',
525 })
526 else:
527 metric_init({
528 'devices': options.devices,
529 })
530
531 while True:
532 for d in descriptors:
533 v = d['call_back'](d['name'])
534 if not options.quiet:
535 print ' %s: %s %s [%s]' % (d['name'], v, d['units'], d['description'])
536
537 if options.gmetric:
538 if d['value_type'] == 'uint':
539 value_type = 'uint32'
540 else:
541 value_type = d['value_type']
542
543 cmd = "%s --conf=%s --value='%s' --units='%s' --type='%s' --name='%s' --slope='%s'" % \
544 (options.gmetric_bin, options.gmond_conf, v, d['units'], value_type, d['name'], d['slope'])
545 os.system(cmd)
546
547 print 'Sleeping 15 seconds'
548 time.sleep(15)