Debian Codegraph @

gmond/python_modules/memcached/memcached.py

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import sys
5 import traceback
6 import os
7 import threading
8 import time
9 import socket
10 import select
11 import errno
12
13 descriptors = list()
14 Desc_Skel = {}
15 _Worker_Thread = None
16 _Lock = threading.Lock() # synchronization lock
17 Debug = False
18
19
20 def dprint(f, *v):
21 if Debug:
22 print >>sys.stderr, "DEBUG: " + f % v
23
24
25 def floatable(str):
26 try:
27 float(str)
28 return True
29 except:
30 return False
31
32
33 class UpdateMetricThread(threading.Thread):
34
35 def __init__(self, params):
36 threading.Thread.__init__(self)
37 self.running = False
38 self.shuttingdown = False
39 self.refresh_rate = 15
40 if "refresh_rate" in params:
41 self.refresh_rate = int(params["refresh_rate"])
42 self.metric = {}
43 self.last_metric = {}
44 self.timeout = 2
45
46 self.host = "localhost"
47 self.port = 11211
48 if "host" in params:
49 self.host = params["host"]
50 if "port" in params:
51 self.port = int(params["port"])
52 self.type = params["type"]
53 self.mp = params["metrix_prefix"]
54
55 def shutdown(self):
56 self.shuttingdown = True
57 if not self.running:
58 return
59 try:
60 self.join()
61 except:
62 pass
63
64 def run(self):
65 self.running = True
66
67 while not self.shuttingdown:
68 _Lock.acquire()
69 self.update_metric()
70 _Lock.release()
71 time.sleep(self.refresh_rate)
72
73 self.running = False
74
75 def update_metric(self):
76 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
77 msg = ""
78 self.last_metric = self.metric.copy()
79 try:
80 dprint("connect %s:%d", self.host, self.port)
81 sock.connect((self.host, self.port))
82 sock.send("stats\r\n")
83
84 while True:
85 try:
86 rfd, wfd, xfd = select.select([sock], [], [], self.timeout)
87
88 if not rfd:
89 print >>sys.stderr, "ERROR: select timeout"
90 break
91
92 for fd in rfd:
93 if fd == sock:
94 try:
95 data = fd.recv(8192)
96 msg += data
97 except (IOError, OSError), e:
98 if e.errno != errno.EINTR:
99 raise
100
101 if msg.find("END"):
102 break
103 except select.error, e:
104 if e[0] != errno.EINTR:
105 raise
106
107 sock.close()
108 except socket.error, e:
109 print >>sys.stderr, "ERROR: %s" % e
110
111 for m in msg.split("\r\n"):
112 d = m.split(" ")
113 if len(d) == 3 and d[0] == "STAT" and floatable(d[2]):
114 self.metric[self.mp + "_" + d[1]] = float(d[2])
115
116 def metric_of(self, name):
117 val = 0
118 mp = name.split("_")[0]
119 if name.rsplit("_", 1)[1] == "rate" and name.rsplit("_", 1)[0] in self.metric:
120 _Lock.acquire()
121 name = name.rsplit("_", 1)[0]
122 if name in self.last_metric:
123 num = self.metric[name] - self.last_metric[name]
124 period = self.metric[mp + "_time"] - self.last_metric[mp + "_time"]
125 try:
126 val = num / period
127 except ZeroDivisionError:
128 val = 0
129 _Lock.release()
130 elif name in self.metric:
131 _Lock.acquire()
132 val = self.metric[name]
133 _Lock.release()
134 # Value should never be negative. If it is counters wrapper due to e.g. memcached restart
135 if val < 0:
136 val = 0
137 return val
138
139
140 def metric_init(params):
141 global descriptors, Desc_Skel, _Worker_Thread, Debug
142
143 print '[memcached] memcached protocol "stats"'
144 if "type" not in params:
145 params["type"] = "memcached"
146
147 if "metrix_prefix" not in params:
148 if params["type"] == "memcached":
149 params["metrix_prefix"] = "mc"
150 elif params["type"] == "Tokyo Tyrant":
151 params["metrix_prefix"] = "tt"
152
153 print params
154
155 # initialize skeleton of descriptors
156 Desc_Skel = {
157 'name' : 'XXX',
158 'call_back' : metric_of,
159 'time_max' : 60,
160 'value_type' : 'float',
161 'format' : '%.0f',
162 'units' : 'XXX',
163 'slope' : 'XXX', # zero|positive|negative|both
164 'description' : 'XXX',
165 'groups' : params["type"],
166 }
167
168 if "refresh_rate" not in params:
169 params["refresh_rate"] = 15
170 if "debug" in params:
171 Debug = params["debug"]
172 dprint("%s", "Debug mode on")
173
174 _Worker_Thread = UpdateMetricThread(params)
175 _Worker_Thread.start()
176
177 # IP:HOSTNAME
178 if "spoof_host" in params:
179 Desc_Skel["spoof_host"] = params["spoof_host"]
180
181 mp = params["metrix_prefix"]
182
183 descriptors.append(create_desc(Desc_Skel, {
184 "name" : mp + "_curr_items",
185 "units" : "items",
186 "slope" : "both",
187 "description": "Current number of items stored",
188 }))
189 descriptors.append(create_desc(Desc_Skel, {
190 "name" : mp + "_cmd_get",
191 "units" : "commands",
192 "slope" : "positive",
193 "description": "Cumulative number of retrieval reqs",
194 }))
195 descriptors.append(create_desc(Desc_Skel, {
196 "name" : mp + "_cmd_set",
197 "units" : "commands",
198 "slope" : "positive",
199 "description": "Cumulative number of storage reqs",
200 }))
201 descriptors.append(create_desc(Desc_Skel, {
202 "name" : mp + "_bytes_read",
203 "units" : "bytes",
204 "slope" : "positive",
205 "description": "Total number of bytes read by this server from network",
206 }))
207 descriptors.append(create_desc(Desc_Skel, {
208 "name" : mp + "_bytes_written",
209 "units" : "bytes",
210 "slope" : "positive",
211 "description": "Total number of bytes sent by this server to network",
212 }))
213 descriptors.append(create_desc(Desc_Skel, {
214 "name" : mp + "_bytes",
215 "units" : "bytes",
216 "slope" : "both",
217 "description": "Current number of bytes used to store items",
218 }))
219 descriptors.append(create_desc(Desc_Skel, {
220 "name" : mp + "_limit_maxbytes",
221 "units" : "bytes",
222 "slope" : "both",
223 "description": "Number of bytes this server is allowed to use for storage",
224 }))
225 descriptors.append(create_desc(Desc_Skel, {
226 "name" : mp + "_curr_connections",
227 "units" : "connections",
228 "slope" : "both",
229 "description": "Number of open connections",
230 }))
231 descriptors.append(create_desc(Desc_Skel, {
232 "name" : mp + "_decr_hits",
233 "units" : "items",
234 "slope" : "positive",
235 "description": "Number of keys that have been decremented and found present ",
236 }))
237 descriptors.append(create_desc(Desc_Skel, {
238 "name" : mp + "_decr_misses",
239 "units" : "items",
240 "slope" : "positive",
241 "description": "Number of items that have been decremented and not found",
242 }))
243 descriptors.append(create_desc(Desc_Skel, {
244 "name" : mp + "_delete_hits",
245 "units" : "items",
246 "slope" : "positive",
247 "description": "Number of keys that have been deleted and found present ",
248 }))
249 descriptors.append(create_desc(Desc_Skel, {
250 "name" : mp + "_delete_misses",
251 "units" : "items",
252 "slope" : "positive",
253 "description": "Number of items that have been deleted and not found",
254 }))
255 descriptors.append(create_desc(Desc_Skel, {
256 "name" : mp + "_evictions",
257 "units" : "items",
258 "slope" : "both",
259 "description": "Number of valid items removed from cache to free memory for new items",
260 }))
261 descriptors.append(create_desc(Desc_Skel, {
262 "name" : mp + "_get_hits",
263 "units" : "items",
264 "slope" : "positive",
265 "description": "Number of keys that have been requested and found present ",
266 }))
267 descriptors.append(create_desc(Desc_Skel, {
268 "name" : mp + "_get_misses",
269 "units" : "items",
270 "slope" : "positive",
271 "description": "Number of items that have been requested and not found",
272 }))
273 descriptors.append(create_desc(Desc_Skel, {
274 "name" : mp + "_get_hits_rate",
275 "units" : "items",
276 "slope" : "both",
277 "description": "Hits per second",
278 }))
279 descriptors.append(create_desc(Desc_Skel, {
280 "name" : mp + "_get_misses_rate",
281 "units" : "items",
282 "slope" : "both",
283 "description": "Misses per second",
284 }))
285 descriptors.append(create_desc(Desc_Skel, {
286 "name" : mp + "_incr_hits",
287 "units" : "items",
288 "slope" : "positive",
289 "description": "Number of keys that have been incremented and found present ",
290 }))
291 descriptors.append(create_desc(Desc_Skel, {
292 "name" : mp + "_incr_misses",
293 "units" : "items",
294 "slope" : "positive",
295 "description": "Number of items that have been incremented and not found",
296 }))
297 descriptors.append(create_desc(Desc_Skel, {
298 "name" : mp + "_cmd_get_rate",
299 "units" : "commands",
300 "slope" : "both",
301 "description": "Gets per second",
302 }))
303 descriptors.append(create_desc(Desc_Skel, {
304 "name" : mp + "_cmd_set_rate",
305 "units" : "commands",
306 "slope" : "both",
307 "description": "Sets per second",
308 }))
309
310 # Tokyo Tyrant
311 if "type" in params and params["type"].lower().find("tokyo") == 0:
312 dtmp = descriptors[:]
313 for d in dtmp:
314 if d["name"] in [
315 mp + "_bytes_read",
316 mp + "_bytes_written",
317 mp + "_limit_maxbytes",
318 mp + "_curr_connections",
319 mp + "_evictions",
320 ]:
321 descriptors.remove(d)
322 for d in descriptors:
323 if d["name"] == mp + "_get_hits":
324 d["name"] = mp + "_cmd_get_hits"
325 if d["name"] == mp + "_get_misses":
326 d["name"] = mp + "_cmd_get_misses"
327
328 descriptors.append(create_desc(Desc_Skel, {
329 "name" : mp + "_cmd_set_hits",
330 "units" : "items",
331 "slope" : "positive",
332 "description": "Number of keys that have been stored and found present ",
333 }))
334 descriptors.append(create_desc(Desc_Skel, {
335 "name" : mp + "_cmd_set_misses",
336 "units" : "items",
337 "slope" : "positive",
338 "description": "Number of items that have been stored and not found",
339 }))
340
341 descriptors.append(create_desc(Desc_Skel, {
342 "name" : mp + "_cmd_delete",
343 "units" : "commands",
344 "slope" : "positive",
345 "description": "Cumulative number of delete reqs",
346 }))
347 descriptors.append(create_desc(Desc_Skel, {
348 "name" : mp + "_cmd_delete_hits",
349 "units" : "items",
350 "slope" : "positive",
351 "description": "Number of keys that have been deleted and found present ",
352 }))
353 descriptors.append(create_desc(Desc_Skel, {
354 "name" : mp + "_cmd_delete_misses",
355 "units" : "items",
356 "slope" : "positive",
357 "description": "Number of items that have been deleted and not found",
358 }))
359
360 return descriptors
361
362
363 def create_desc(skel, prop):
364 d = skel.copy()
365 for k, v in prop.iteritems():
366 d[k] = v
367 return d
368
369
370 def metric_of(name):
371 return _Worker_Thread.metric_of(name)
372
373
374 def metric_cleanup():
375 _Worker_Thread.shutdown()
376
377
378 if __name__ == '__main__':
379 try:
380 params = {
381 "host" : "localhost",
382 "port" : 11211,
383 # "host" : "tt101",
384 # "port" : 1978,
385 # "type" : "Tokyo Tyrant",
386 # "metrix_prefix" : "tt101",
387 "debug" : True,
388 }
389 metric_init(params)
390
391 while True:
392 for d in descriptors:
393 v = d['call_back'](d['name'])
394 print ('value for %s is ' + d['format']) % (d['name'], v)
395 time.sleep(5)
396 except KeyboardInterrupt:
397 time.sleep(0.2)
398 os._exit(1)
399 except:
400 traceback.print_exc()
401 os._exit(1)