Debian Codegraph @

gmond/python_modules/process/procstat.py

raw
1 ### This script reports process metrics to ganglia.
2 ###
3 ### Notes:
4 ### This script exposes values for CPU and memory utilization
5 ### for running processes. You can retrieve the process ID from
6 ### either providing a pidfile or an awk regular expression.
7 ### Using a pidfile is the most efficient and direct method.
8 ###
9 ### When using a regular expression, keep in mind that there is
10 ### a chance for a false positive. This script will help to avoid
11 ### these by only returning parent processes. This means that
12 ### the results are limited to processes where ppid = 1.
13 ###
14 ### This script also comes with the ability to test your regular
15 ### expressions via command line arguments "-t".
16 ###
17 ### Testing:
18 ### -- This is a correct examples of how to monitor apache.
19 ###
20 ### $ python procstat.py -p httpd -v '/var/run/httpd.pid' -t
21 ### Testing httpd: /var/run/httpd.pid
22 ### Processes in this group:
23 ### PID, ARGS
24 ### 11058 /usr/sbin/httpd
25 ### 8817 /usr/sbin/httpd
26 ### 9000 /usr/sbin/httpd
27 ### 9001 /usr/sbin/httpd
28 ###
29 ### waiting 2 seconds
30 ### procstat_httpd_mem: 202076 KB [The total memory utilization]
31 ### procstat_httpd_cpu: 0.3 percent [The total percent CPU utilization]
32 ###
33 ### -- This example shows a regex that returns no processes with a
34 ### ppid of 1.
35 ###
36 ### $ python procstat.py -p test -v 'wrong' -t
37 ### Testing test: wrong
38 ### failed getting pgid: no process returned
39 ### ps -Ao pid,ppid,pgid,args | awk 'wrong && $2 == 1 && !/awk/ && !/procstat\.py/ {print $0}'
40 ###
41 ### -- This example shows a regex that returns more than one process
42 ### with a ppid of 1.
43 ###
44 ### $ python procstat.py -p test -v '/mingetty/' -t
45 ### Testing test: /mingetty/
46 ### failed getting pgid: more than 1 result returned
47 ### ps -Ao pid,ppid,pgid,args | awk '/mingetty/ && $2 == 1 && !/awk/ && !/procstat\.py/ {print $0}'
48 ### 7313 1 7313 /sbin/mingetty tty1
49 ### 7314 1 7314 /sbin/mingetty tty2
50 ### 7315 1 7315 /sbin/mingetty tty3
51 ### 7316 1 7316 /sbin/mingetty tty4
52 ### 7317 1 7317 /sbin/mingetty tty5
53 ### 7318 1 7318 /sbin/mingetty tty6
54 ###
55 ### Command Line Example:
56 ### $ python procstat.py -p httpd,opennms,splunk,splunk-web \
57 ### -v '/var/run/httpd.pid','/opt/opennms/logs/daemon/opennms.pid','/splunkd.*start/','/twistd.*SplunkWeb/'
58 ###
59 ### procstat_httpd_mem: 202068 KB [The total memory utilization]
60 ### procstat_splunk_mem: 497848 KB [The total memory utilization]
61 ### procstat_splunk-web_mem: 32636 KB [The total memory utilization]
62 ### procstat_opennms_mem: 623112 KB [The total memory utilization]
63 ### procstat_httpd_cpu: 0.3 percent [The total percent CPU utilization]
64 ### procstat_splunk_cpu: 0.6 percent [The total percent CPU utilization]
65 ### procstat_splunk-web_cpu: 0.1 percent [The total percent CPU utilization]
66 ### procstat_opennms_cpu: 7.1 percent [The total percent CPU utilization]
67 ###
68 ### Example Values:
69 ### httpd: /var/run/httpd.pid or \/usr\/sbin\/httpd
70 ### mysqld: /var/run/mysqld/mysqld.pid or /\/usr\/bin\/mysqld_safe/
71 ### postgresql: /var/run/postmaster.[port].pid or /\/usr\/bin\/postmaster.*[port]/
72 ### splunk: /splunkd.*start/
73 ### splunk-web: /twistd.*SplunkWeb/
74 ### opennms: /opt/opennms/logs/daemon/opennms.pid or java.*Dopennms
75 ### netflow: /java.*NetFlow/
76 ### postfix: /var/spool/postfix/pid/master.pid or /\/usr\/libexec\/postfix\/master/
77 ###
78 ### Error Tests:
79 ### python procstat.py -p test-more,test-none,test-pidfail -v '/java/','/javaw/','java.pid' -t
80 ###
81 ### Changelog:
82 ### v1.0.1 - 2010-07-23
83 ### * Initial version
84 ###
85 ### v1.1.0 - 2010-07-28
86 ### * Modified the process regex search to find the parent
87 ### process and then find all processes with the same process
88 ### group ID (pgid). "ps" is only used for regex searching on
89 ### the initial lookup for the parent pid (ppid). Now all
90 ### subsequent calls use /proc/[pid]/stat for CPU jiffies, and
91 ### /proc/[pid]/statm for memory rss.
92 ### * Added testing switch "-t" to help troubleshoot a regex
93 ### * Added display switches "-s" and "-m" to format the output
94 ### of /proc/[pid]/stat and /proc/[pid]/statm
95 ###
96
97 ### Copyright Jamie Isaacs. 2010
98 ### License to use, modify, and distribute under the GPL
99 ### http://www.gnu.org/licenses/gpl.txt
100
101 import glob
102 import logging
103 import os.path
104 import subprocess
105 import sys
106 import time
107 import traceback
108
109 descriptors = []
110
111 logging.basicConfig(level=logging.ERROR, format="%(asctime)s - %(name)s - %(levelname)s\t Thread-%(thread)d - %(message)s", filename='/tmp/gmond.log', filemode='w')
112 logging.debug('starting up')
113
114 last_update = 0
115 stats = {}
116 last_val = {}
117 pgid_list = {}
118
119 MAX_UPDATE_TIME = 15
120
121 # clock ticks per second... jiffies (HZ)
122 JIFFIES_PER_SEC = os.sysconf('SC_CLK_TCK')
123
124 PAGE_SIZE = os.sysconf('SC_PAGE_SIZE')
125
126 PROCESSES = {}
127
128
129 def readCpu(pid):
130 try:
131 stat = file('/proc/' + pid + '/stat', 'rt').readline().split()
132 #logging.debug(' stat (' + pid + '): ' + str(stat))
133 utime = int(stat[13])
134 stime = int(stat[14])
135 cutime = int(stat[15])
136 cstime = int(stat[16])
137 return (utime + stime + cutime + cstime)
138 except:
139 logging.warning('failed to get (' + str(pid) + ') stats')
140 return 0
141
142
143 def get_pgid(proc):
144 logging.debug('getting pgid for process: ' + proc)
145 ERROR = 0
146
147 if pgid_list.has_key(proc) and os.path.exists('/proc/' + pgid_list[proc][0]):
148 return pgid_list[proc]
149
150 val = PROCESSES[proc]
151 # Is this a pidfile? Last 4 chars are .pid
152 if '.pid' in val[-4:]:
153 if os.path.exists(val):
154 logging.debug(' pidfile found')
155 ppid = file(val, 'rt').readline().strip()
156 pgid = file('/proc/' + ppid + '/stat', 'rt').readline().split()[4]
157 else:
158 raise Exception('pidfile (' + val + ') does not exist')
159
160 else:
161 # This is a regex, lets search for it
162 regex = PROCESSES[proc]
163 cmd = "ps -Ao pid,ppid,pgid,args | awk '" + regex + " && $2 == 1 && !/awk/ && !/procstat\.py/ {print $0}'"
164 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
165 out, err = p.communicate()
166
167 if p.returncode:
168 raise Exception('failed executing ps\n' + cmd + '\n' + err)
169
170 result = out.strip().split('\n')
171 logging.debug(' result: ' + str(result))
172
173 if len(result) > 1:
174 raise Exception('more than 1 result returned\n' + cmd + '\n' + out.strip())
175
176 if result[0] in '':
177 raise Exception('no process returned\n' + cmd)
178
179 res = result[0].split()
180 ppid = res[0]
181 pgid = res[2]
182
183 if os.path.exists('/proc/' + ppid):
184 logging.debug(' ppid: ' + ppid + ' pgid: ' + pgid)
185 return (ppid, pgid)
186 else:
187 return ERROR
188
189
190 def get_pgroup(ppid, pgid):
191 '''Return a list of pids having the same pgid, with the first in the list being the parent pid.'''
192 logging.debug('getting pids for ppid/pgid: ' + ppid + '/' + pgid)
193
194 # Get all processes in this group
195 p_list = []
196 for stat_file in glob.glob('/proc/[1-9]*/stat'):
197 try:
198 stat = file(stat_file, 'rt').readline().split()
199 if stat[4] == pgid:
200 p_list.append(stat[0])
201 except:
202 # likely the pid has exited. this is normal.
203 pass
204
205 # place pid at the top of the list
206 if ppid in p_list:
207 p_list.remove(ppid)
208 p_list.insert(0, ppid)
209 else:
210 logging.warning('failed to find ppid ' + str(ppid) + ' in p_list')
211
212 logging.debug('p_list: ' + str(p_list))
213
214 if not len(p_list):
215 logging.warning('failed getting pids')
216
217 return p_list
218
219
220 def get_rss(pids):
221 logging.debug('getting rss for pids')
222
223 rss = 0
224 for p in pids:
225 try:
226 statm = open('/proc/' + p + '/statm', 'rt').readline().split()
227 #logging.debug(' statm (' + p + '): ' + str(statm))
228 except:
229 # Process finished, ignore this mem usage
230 logging.warning(' failed getting statm for pid: ' + p)
231 continue
232
233 rss += int(statm[1])
234
235 rss *= PAGE_SIZE
236 return rss
237
238
239 def test(params):
240 global PROCESSES, MAX_UPDATE_TIME
241
242 MAX_UPDATE_TIME = 2
243
244 logging.debug('testing processes: ' + str(params))
245
246 PROCESSES = params
247
248 for proc, val in PROCESSES.items():
249 print('')
250 print(' Testing ' + proc + ': ' + val)
251
252 try:
253 (ppid, pgid) = get_pgid(proc)
254 except Exception, e:
255 print(' failed getting pgid: ' + str(e))
256 continue
257
258 pids = get_pgroup(ppid, pgid)
259
260 print(' Processes in this group: ')
261 print(' PID, ARGS')
262 for pid in pids:
263 # Read from binary file containing command line arguments
264 args = file('/proc/' + pid + '/cmdline', 'rt').readline().replace('\0', ' ')
265 print(' ' + pid + ' ' + args)
266
267 logging.debug('success testing')
268
269
270 def update_stats():
271 logging.debug('updating stats')
272 global last_update, stats, last_val
273
274 cur_time = time.time()
275
276 if cur_time - last_update < MAX_UPDATE_TIME:
277 logging.debug(' wait ' + str(int(MAX_UPDATE_TIME - (cur_time - last_update))) + ' seconds')
278 return True
279 else:
280 last_update = cur_time
281
282 for proc, val in PROCESSES.items():
283 logging.debug(' updating for ' + proc)
284
285 # setup storage lists
286 if not proc in stats:
287 stats[proc] = {}
288 if not proc in last_val:
289 last_val[proc] = {}
290
291 #####
292 # Update CPU utilization
293 try:
294 (ppid, pgid) = get_pgid(proc)
295 except Exception, e:
296 logging.warning(' failed getting pgid: ' + str(e))
297 stats[proc]['cpu'] = 0.0
298 stats[proc]['mem'] = 0
299 continue
300
301 # save for later
302 pgid_list[proc] = (ppid, pgid)
303
304 pids = get_pgroup(ppid, pgid)
305
306 cpu_time = time.time()
307 proc_time = 0
308 for p in pids:
309 proc_time += readCpu(p)
310
311 logging.debug(' proc_time: ' + str(proc_time) + ' cpu_time: ' + str(cpu_time))
312
313 # do we have an old value to calculate with?
314 if 'cpu_time' in last_val[proc]:
315 logging.debug(' last_val: ' + str(last_val[proc]))
316 logging.debug(' calc: 100 * ' + str(proc_time - last_val[proc]['proc_time']) + ' / ' + str(cpu_time - last_val[proc]['cpu_time']) + ' * ' + str(JIFFIES_PER_SEC))
317 stats[proc]['cpu'] = 100 * (proc_time - last_val[proc]['proc_time']) / float((cpu_time - last_val[proc]['cpu_time']) * JIFFIES_PER_SEC)
318
319 logging.debug(' calc: ' + str(stats[proc]['cpu']))
320 else:
321 stats[proc]['cpu'] = 0.0
322
323 last_val[proc]['cpu_time'] = cpu_time
324 last_val[proc]['proc_time'] = proc_time
325
326 #####
327 # Update Mem utilization
328 rss = get_rss(pids)
329 stats[proc]['mem'] = rss
330
331 logging.debug('success refreshing stats')
332 logging.debug('stats: ' + str(stats))
333
334 return True
335
336
337 def get_stat(name):
338 logging.debug('getting stat: ' + name)
339
340 ret = update_stats()
341
342 if ret:
343 if name.startswith('procstat_'):
344 nsp = name.split('_')
345 proc = '_'.join(nsp[1:-1])
346 label = nsp[-1]
347
348 try:
349 return stats[proc][label]
350 except:
351 logging.warning('failed to fetch [' + proc + '] ' + name)
352 return 0
353 else:
354 label = name
355
356 try:
357 return stats[label]
358 except:
359 logging.warning('failed to fetch ' + name)
360 return 0
361 else:
362 return 0
363
364
365 def metric_init(params):
366 global descriptors
367 global PROCESSES
368
369 logging.debug('init: ' + str(params))
370
371 PROCESSES = params
372
373 #for proc,regex in PROCESSES.items():
374
375 update_stats()
376
377 descriptions = dict(
378 cpu = {
379 'units': 'percent',
380 'value_type': 'float',
381 'format': '%.1f',
382 'description': 'The total percent CPU utilization'},
383
384 mem = {
385 'units': 'B',
386 'description': 'The total memory utilization'}
387 )
388
389 time_max = 60
390 for label in descriptions:
391 for proc in PROCESSES:
392 if stats[proc].has_key(label):
393
394 d = {
395 'name': 'procstat_' + proc + '_' + label,
396 'call_back': get_stat,
397 'time_max': time_max,
398 'value_type': 'uint',
399 'units': '',
400 'slope': 'both',
401 'format': '%u',
402 'description': label,
403 'groups': 'procstat'
404 }
405
406 # Apply metric customizations from descriptions
407 d.update(descriptions[label])
408
409 descriptors.append(d)
410
411 else:
412 logging.error("skipped " + proc + '_' + label)
413
414 #logging.debug('descriptors: ' + str(descriptors))
415
416 return descriptors
417
418
419 def display_proc_stat(pid):
420 try:
421 stat = file('/proc/' + pid + '/stat', 'rt').readline().split()
422
423 fields = [
424 'pid', 'comm', 'state', 'ppid', 'pgrp', 'session',
425 'tty_nr', 'tty_pgrp', 'flags', 'min_flt', 'cmin_flt', 'maj_flt',
426 'cmaj_flt', 'utime', 'stime', 'cutime', 'cstime', 'priority',
427 'nice', 'num_threads', 'it_real_value', 'start_time', 'vsize', 'rss',
428 'rlim', 'start_code', 'end_code', 'start_stack', 'esp', 'eip',
429 'pending', 'blocked', 'sigign', 'sigcatch', 'wchan', 'nswap',
430 'cnswap', 'exit_signal', 'processor', 'rt_priority', 'policy'
431 ]
432
433 # Display them
434 i = 0
435 for f in fields:
436 print '%15s: %s' % (f, stat[i])
437 i += 1
438
439 except:
440 print('failed to get /proc/' + pid + '/stat')
441 print(traceback.print_exc(file=sys.stdout))
442
443
444 def display_proc_statm(pid):
445 try:
446 statm = file('/proc/' + pid + '/statm', 'rt').readline().split()
447
448 fields = [
449 'size', 'rss', 'share', 'trs', 'drs', 'lrs', 'dt'
450 ]
451
452 # Display them
453 i = 0
454 for f in fields:
455 print '%15s: %s' % (f, statm[i])
456 i += 1
457
458 except:
459 print('failed to get /proc/' + pid + '/statm')
460 print(traceback.print_exc(file=sys.stdout))
461
462
463 def metric_cleanup():
464 logging.shutdown()
465 # pass
466
467 if __name__ == '__main__':
468 from optparse import OptionParser
469 import os
470
471 logging.debug('running from cmd line')
472 parser = OptionParser()
473 parser.add_option('-p', '--processes', dest='processes', default='', help='processes to explicitly check')
474 parser.add_option('-v', '--value', dest='value', default='', help='regex or pidfile for each processes')
475 parser.add_option('-s', '--stat', dest='stat', default='', help='display the /proc/[pid]/stat file for this pid')
476 parser.add_option('-m', '--statm', dest='statm', default='', help='display the /proc/[pid]/statm file for this pid')
477 parser.add_option('-b', '--gmetric-bin', dest='gmetric_bin', default='/usr/bin/gmetric', help='path to gmetric binary')
478 parser.add_option('-c', '--gmond-conf', dest='gmond_conf', default='/etc/ganglia/gmond.conf', help='path to gmond.conf')
479 parser.add_option('-g', '--gmetric', dest='gmetric', action='store_true', default=False, help='submit via gmetric')
480 parser.add_option('-q', '--quiet', dest='quiet', action='store_true', default=False)
481 parser.add_option('-t', '--test', dest='test', action='store_true', default=False, help='test the regex list')
482
483 (options, args) = parser.parse_args()
484
485 if options.stat != '':
486 display_proc_stat(options.stat)
487 sys.exit(0)
488 elif options.statm != '':
489 display_proc_statm(options.statm)
490 sys.exit(0)
491
492 _procs = options.processes.split(',')
493 _val = options.value.split(',')
494 params = {}
495 i = 0
496 for proc in _procs:
497 params[proc] = _val[i]
498 i += 1
499
500 if options.test:
501 test(params)
502 update_stats()
503
504 print('')
505 print(' waiting ' + str(MAX_UPDATE_TIME) + ' seconds')
506 time.sleep(MAX_UPDATE_TIME)
507
508 metric_init(params)
509
510 for d in descriptors:
511 v = d['call_back'](d['name'])
512 if not options.quiet:
513 print ' %s: %s %s [%s]' % (d['name'], d['format'] % v, d['units'], d['description'])
514
515 if options.gmetric:
516 if d['value_type'] == 'uint':
517 value_type = 'uint32'
518 else:
519 value_type = d['value_type']
520
521 cmd = "%s --conf=%s --value='%s' --units='%s' --type='%s' --name='%s' --slope='%s'" % \
522 (options.gmetric_bin, options.gmond_conf, v, d['units'], value_type, d['name'], d['slope'])
523 os.system(cmd)