Debian Codegraph @

check_mongodb/check_mongodb.py

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 #!/usr/bin/env python
2
3 #
4 # A MongoDB Nagios check script
5 #
6
7 # Script idea taken from a Tag1 script I found and I modified it a lot
8 #
9 # Main Author
10 # - Mike Zupan <[email protected]>
11 # Contributers
12 # - Frank Brandewiede <[email protected]> <[email protected]> <[email protected]>
13 # - Sam Perman <[email protected]>
14 # - Shlomo Priymak <[email protected]>
15 # - @jhoff909 on github
16 # - @jbraeuer on github
17 # - Dag Stockstad <[email protected]>
18 # - @Andor on github
19 # - Steven Richards - Captainkrtek on github
20 # - Max Vernimmen - @mvernimmen-CG / @mvernimmen on github
21 # - Kris Nova - @[email protected] github.com/kris-nova
22 # - Jan Kantert - [email protected]
23 #
24 # USAGE
25 #
26 # See the README.md
27 #
28
29 from __future__ import print_function
30 from __future__ import division
31 import sys
32 import time
33 import optparse
34 import re
35 import os
36 import numbers
37 import socket
38
39 try:
40 import pymongo
41 except ImportError as e:
42 print(e)
43 sys.exit(2)
44
45 # As of pymongo v 1.9 the SON API is part of the BSON package, therefore attempt
46 # to import from there and fall back to pymongo in cases of older pymongo
47 if pymongo.version >= "1.9":
48 import bson.son as son
49 else:
50 import pymongo.son as son
51
52
53 #
54 # thanks to http://stackoverflow.com/a/1229667/72987
55 #
56 def optional_arg(arg_default):
57 def func(option, opt_str, value, parser):
58 if parser.rargs and not parser.rargs[0].startswith('-'):
59 val = parser.rargs[0]
60 parser.rargs.pop(0)
61 else:
62 val = arg_default
63 setattr(parser.values, option.dest, val)
64 return func
65
66
67 def performance_data(perf_data, params):
68 data = ''
69 if perf_data:
70 data = " |"
71 for p in params:
72 p += (None, None, None, None)
73 param, param_name, warning, critical = p[0:4]
74 data += "%s=%s" % (param_name, str(param))
75 if warning or critical:
76 warning = warning or 0
77 critical = critical or 0
78 data += ";%s;%s" % (warning, critical)
79
80 data += " "
81
82 return data
83
84
85 def numeric_type(param):
86 return param is None or isinstance(param, numbers.Real)
87
88
89 def check_levels(param, warning, critical, message, ok=[]):
90 if (numeric_type(critical) and numeric_type(warning)):
91 if param >= critical:
92 print("CRITICAL - " + message)
93 sys.exit(2)
94 elif param >= warning:
95 print("WARNING - " + message)
96 sys.exit(1)
97 else:
98 print("OK - " + message)
99 sys.exit(0)
100 else:
101 if param in critical:
102 print("CRITICAL - " + message)
103 sys.exit(2)
104
105 if param in warning:
106 print("WARNING - " + message)
107 sys.exit(1)
108
109 if param in ok:
110 print("OK - " + message)
111 sys.exit(0)
112
113 # unexpected param value
114 print("CRITICAL - Unexpected value : %d" % param + "; " + message)
115 return 2
116
117
118 def get_server_status(con):
119 try:
120 set_read_preference(con.admin)
121 data = con.admin.command(pymongo.son_manipulator.SON([('serverStatus', 1)]))
122 except:
123 data = con.admin.command(son.SON([('serverStatus', 1)]))
124 return data
125
126 def split_host_port(string):
127 if not string.rsplit(':', 1)[-1].isdigit():
128 return (string, None)
129 string = string.rsplit(':', 1)
130 host = string[0] # 1st index is always host
131 port = int(string[1])
132 return (host, port)
133
134
135 def main(argv):
136 p = optparse.OptionParser(conflict_handler="resolve", description="This Nagios plugin checks the health of mongodb.")
137
138 p.add_option('-H', '--host', action='store', type='string', dest='host', default='127.0.0.1', help='The hostname you want to connect to')
139 p.add_option('-h', '--host-to-check', action='store', type='string', dest='host_to_check', default=None, help='The hostname you want to check (if this is different from the host you are connecting)')
140 p.add_option('--rdns-lookup', action='store_true', dest='rdns_lookup', default=False, help='RDNS(PTR) lookup on given host/host-to-check, to convert ip-address to fqdn')
141 p.add_option('-P', '--port', action='store', type='int', dest='port', default=27017, help='The port mongodb is running on')
142 p.add_option('--port-to-check', action='store', type='int', dest='port_to_check', default=None, help='The port you want to check (if this is different from the port you are connecting)')
143 p.add_option('-u', '--user', action='store', type='string', dest='user', default=None, help='The username you want to login as')
144 p.add_option('-p', '--pass', action='store', type='string', dest='passwd', default=None, help='The password you want to use for that user')
145 p.add_option('-W', '--warning', action='store', dest='warning', default=None, help='The warning threshold you want to set')
146 p.add_option('-C', '--critical', action='store', dest='critical', default=None, help='The critical threshold you want to set')
147 p.add_option('-A', '--action', action='store', type='choice', dest='action', default='connect', help='The action you want to take',
148 choices=['connect', 'connections', 'replication_lag', 'replication_lag_percent', 'replset_state', 'memory', 'memory_mapped', 'lock',
149 'flushing', 'last_flush_time', 'index_miss_ratio', 'databases', 'collections', 'database_size', 'database_indexes', 'collection_documents', 'collection_indexes', 'collection_size',
150 'collection_storageSize', 'queues', 'oplog', 'journal_commits_in_wl', 'write_data_files', 'journaled', 'opcounters', 'current_lock', 'replica_primary',
151 'page_faults', 'asserts', 'queries_per_second', 'page_faults', 'chunks_balance', 'connect_primary', 'collection_state', 'row_count', 'replset_quorum'])
152 p.add_option('--max-lag', action='store_true', dest='max_lag', default=False, help='Get max replication lag (for replication_lag action only)')
153 p.add_option('--mapped-memory', action='store_true', dest='mapped_memory', default=False, help='Get mapped memory instead of resident (if resident memory can not be read)')
154 p.add_option('-D', '--perf-data', action='store_true', dest='perf_data', default=False, help='Enable output of Nagios performance data')
155 p.add_option('-d', '--database', action='store', dest='database', default='admin', help='Specify the database to check')
156 p.add_option('--all-databases', action='store_true', dest='all_databases', default=False, help='Check all databases (action database_size)')
157 p.add_option('-s', '--ssl', dest='ssl', default=False, action='callback', callback=optional_arg(True), help='Connect using SSL')
158 p.add_option('-r', '--replicaset', dest='replicaset', default=None, action='callback', callback=optional_arg(True), help='Connect to replicaset')
159 p.add_option('-q', '--querytype', action='store', dest='query_type', default='query', help='The query type to check [query|insert|update|delete|getmore|command] from queries_per_second')
160 p.add_option('-c', '--collection', action='store', dest='collection', default='admin', help='Specify the collection to check')
161 p.add_option('-T', '--time', action='store', type='int', dest='sample_time', default=1, help='Time used to sample number of pages faults')
162 p.add_option('-M', '--mongoversion', action='store', type='choice', dest='mongo_version', default='2', help='The MongoDB version you are talking with, either 2 or 3',
163 choices=['2','3'])
164 p.add_option('-a', '--authdb', action='store', type='string', dest='authdb', default='admin', help='The database you want to authenticate against')
165 p.add_option('--insecure', action='store_true', dest='insecure', default=False, help="Don't verify SSL/TLS certificates")
166 p.add_option('--ssl-ca-cert-file', action='store', type='string', dest='ssl_ca_cert_file', default=None, help='Path to Certificate Authority file for SSL')
167 p.add_option('-f', '--ssl-cert-file', action='store', type='string', dest='cert_file', default=None, help='Path to PEM encoded key and cert for client authentication')
168 p.add_option('-m','--auth-mechanism', action='store', type='choice', dest='auth_mechanism', default=None, help='Auth mechanism used for auth with mongodb',
169 choices=['MONGODB-X509','SCRAM-SHA-256','SCRAM-SHA-1'])
170 p.add_option('--disable_retry_writes', dest='retry_writes_disabled', default=False, action='callback', callback=optional_arg(True), help='Disable retryWrites feature')
171
172 options, arguments = p.parse_args()
173 host = options.host
174 host_to_check = options.host_to_check if options.host_to_check else options.host
175 rdns_lookup = options.rdns_lookup
176 if (rdns_lookup):
177 host_to_check = socket.getnameinfo((host_to_check, 0), 0)[0]
178 port = options.port
179 port_to_check = options.port_to_check if options.port_to_check else options.port
180 user = options.user
181 passwd = options.passwd
182 authdb = options.authdb
183
184 query_type = options.query_type
185 collection = options.collection
186 sample_time = options.sample_time
187 if (options.action == 'replset_state'):
188 warning = str(options.warning or "")
189 critical = str(options.critical or "")
190 else:
191 warning = float(options.warning or 0)
192 critical = float(options.critical or 0)
193
194 action = options.action
195 perf_data = options.perf_data
196 max_lag = options.max_lag
197 mongo_version = options.mongo_version
198 database = options.database
199 ssl = options.ssl
200 replicaset = options.replicaset
201 insecure = options.insecure
202 ssl_ca_cert_file = options.ssl_ca_cert_file
203 cert_file = options.cert_file
204 auth_mechanism = options.auth_mechanism
205 retry_writes_disabled = options.retry_writes_disabled
206
207 if action == 'replica_primary' and replicaset is None:
208 return "replicaset must be passed in when using replica_primary check"
209 elif not action == 'replica_primary' and replicaset:
210 return "passing a replicaset while not checking replica_primary does not work"
211
212 #
213 # moving the login up here and passing in the connection
214 #
215 start = time.time()
216 err, con = mongo_connect(host, port, ssl, user, passwd, replicaset, authdb, insecure, ssl_ca_cert_file, cert_file, auth_mechanism, retry_writes_disabled=retry_writes_disabled)
217 if err != 0:
218 return err
219
220 # Autodetect mongo-version and force pymongo to let us know if it can connect or not.
221 err, mongo_version = check_version(con)
222 if err != 0:
223 return err
224
225 conn_time = time.time() - start
226
227 if action == "connections":
228 return check_connections(con, warning, critical, perf_data)
229 elif action == "replication_lag":
230 return check_rep_lag(con, host_to_check, port_to_check, rdns_lookup, warning, critical, False, perf_data, max_lag, ssl, user, passwd, replicaset, authdb, insecure, ssl_ca_cert_file, cert_file, auth_mechanism, retry_writes_disabled=retry_writes_disabled)
231 elif action == "replication_lag_percent":
232 return check_rep_lag(con, host_to_check, port_to_check, rdns_lookup, warning, critical, True, perf_data, max_lag, ssl, user, passwd, replicaset, authdb, insecure, ssl_ca_cert_file, cert_file, auth_mechanism, retry_writes_disabled=retry_writes_disabled)
233 elif action == "replset_state":
234 return check_replset_state(con, perf_data, warning, critical)
235 elif action == "memory":
236 return check_memory(con, warning, critical, perf_data, options.mapped_memory, host)
237 elif action == "memory_mapped":
238 return check_memory_mapped(con, warning, critical, perf_data)
239 elif action == "queues":
240 return check_queues(con, warning, critical, perf_data)
241 elif action == "lock":
242 return check_lock(con, warning, critical, perf_data, mongo_version)
243 elif action == "current_lock":
244 return check_current_lock(con, host, port, warning, critical, perf_data)
245 elif action == "flushing":
246 return check_flushing(con, warning, critical, True, perf_data)
247 elif action == "last_flush_time":
248 return check_flushing(con, warning, critical, False, perf_data)
249 elif action == "index_miss_ratio":
250 index_miss_ratio(con, warning, critical, perf_data)
251 elif action == "databases":
252 return check_databases(con, warning, critical, perf_data)
253 elif action == "collections":
254 return check_collections(con, warning, critical, perf_data)
255 elif action == "oplog":
256 return check_oplog(con, warning, critical, perf_data)
257 elif action == "journal_commits_in_wl":
258 return check_journal_commits_in_wl(con, warning, critical, perf_data)
259 elif action == "database_size":
260 if options.all_databases:
261 return check_all_databases_size(con, warning, critical, perf_data)
262 else:
263 return check_database_size(con, database, warning, critical, perf_data)
264 elif action == "database_indexes":
265 return check_database_indexes(con, database, warning, critical, perf_data)
266 elif action == "collection_documents":
267 return check_collection_documents(con, database, collection, warning, critical, perf_data)
268 elif action == "collection_indexes":
269 return check_collection_indexes(con, database, collection, warning, critical, perf_data)
270 elif action == "collection_size":
271 return check_collection_size(con, database, collection, warning, critical, perf_data)
272 elif action == "collection_storageSize":
273 return check_collection_storageSize(con, database, collection, warning, critical, perf_data)
274 elif action == "journaled":
275 return check_journaled(con, warning, critical, perf_data)
276 elif action == "write_data_files":
277 return check_write_to_datafiles(con, warning, critical, perf_data)
278 elif action == "opcounters":
279 return check_opcounters(con, host, port, warning, critical, perf_data)
280 elif action == "asserts":
281 return check_asserts(con, host, port, warning, critical, perf_data)
282 elif action == "replica_primary":
283 return check_replica_primary(con, host, warning, critical, perf_data, replicaset, mongo_version)
284 elif action == "queries_per_second":
285 return check_queries_per_second(con, query_type, warning, critical, perf_data, mongo_version)
286 elif action == "page_faults":
287 check_page_faults(con, sample_time, warning, critical, perf_data)
288 elif action == "chunks_balance":
289 chunks_balance(con, database, collection, warning, critical)
290 elif action == "connect_primary":
291 return check_connect_primary(con, warning, critical, perf_data)
292 elif action == "collection_state":
293 return check_collection_state(con, database, collection)
294 elif action == "row_count":
295 return check_row_count(con, database, collection, warning, critical, perf_data)
296 elif action == "replset_quorum":
297 return check_replset_quorum(con, perf_data)
298 else:
299 return check_connect(host, port, warning, critical, perf_data, user, passwd, conn_time)
300
301
302 def mongo_connect(host=None, port=None, ssl=False, user=None, passwd=None, replica=None, authdb="admin", insecure=False, ssl_ca_cert_file=None, ssl_cert=None, auth_mechanism=None, retry_writes_disabled=False):
303 from pymongo.errors import ConnectionFailure
304 from pymongo.errors import PyMongoError
305 import ssl as SSL
306
307 con_args = dict()
308
309 if ssl:
310 if insecure:
311 con_args['ssl_cert_reqs'] = SSL.CERT_NONE
312 else:
313 con_args['ssl_cert_reqs'] = SSL.CERT_REQUIRED
314 con_args['ssl'] = ssl
315 if ssl_ca_cert_file:
316 con_args['ssl_ca_certs'] = ssl_ca_cert_file
317 if ssl_cert:
318 con_args['ssl_certfile'] = ssl_cert
319
320 if retry_writes_disabled:
321 con_args['retryWrites'] = False
322
323 try:
324 # ssl connection for pymongo > 2.3
325 if pymongo.version >= "2.3":
326 if replica is None:
327 con = pymongo.MongoClient(host, port, **con_args)
328 else:
329 con = pymongo.MongoClient(host, port, read_preference=pymongo.ReadPreference.SECONDARY, replicaSet=replica, **con_args)
330 else:
331 if replica is None:
332 con = pymongo.Connection(host, port, slave_okay=True, network_timeout=10)
333 else:
334 con = pymongo.Connection(host, port, slave_okay=True, network_timeout=10)
335
336 # we must authenticate the connection, otherwise we won't be able to perform certain operations
337 if ssl_cert and ssl_ca_cert_file and user and auth_mechanism == 'SCRAM-SHA-256':
338 con.the_database.authenticate(user, mechanism='SCRAM-SHA-256')
339 elif ssl_cert and ssl_ca_cert_file and user and auth_mechanism == 'SCRAM-SHA-1':
340 con.the_database.authenticate(user, mechanism='SCRAM-SHA-1')
341 elif ssl_cert and ssl_ca_cert_file and user and auth_mechanism == 'MONGODB-X509':
342 con.the_database.authenticate(user, mechanism='MONGODB-X509')
343
344 try:
345 result = con.admin.command("ismaster")
346 except ConnectionFailure:
347 print("CRITICAL - Connection to Mongo server on %s:%s has failed" % (host, port) )
348 sys.exit(2)
349
350 if 'arbiterOnly' in result and result['arbiterOnly'] == True:
351 print("OK - State: 7 (Arbiter on port %s)" % (port))
352 sys.exit(0)
353
354 if user and passwd:
355 db = con[authdb]
356 try:
357 db.authenticate(user, password=passwd)
358 except PyMongoError:
359 sys.exit("Username/Password incorrect")
360
361 # Ping to check that the server is responding.
362 con.admin.command("ping")
363
364 except Exception as e:
365 if isinstance(e, pymongo.errors.AutoReconnect) and str(e).find(" is an arbiter") != -1:
366 # We got a pymongo AutoReconnect exception that tells us we connected to an Arbiter Server
367 # This means: Arbiter is reachable and can answer requests/votes - this is all we need to know from an arbiter
368 print("OK - State: 7 (Arbiter)")
369 sys.exit(0)
370 return exit_with_general_critical(e), None
371 return 0, con
372
373
374 def exit_with_general_warning(e):
375 if isinstance(e, SystemExit):
376 return e
377 else:
378 print("WARNING - General MongoDB warning:", e)
379 return 1
380
381
382 def exit_with_general_critical(e):
383 if isinstance(e, SystemExit):
384 return e
385 else:
386 print("CRITICAL - General MongoDB Error:", e)
387 return 2
388
389
390 def set_read_preference(db):
391 if pymongo.version >= "2.2":
392 pymongo.read_preferences.Secondary
393 else:
394 db.read_preference = pymongo.ReadPreference.SECONDARY
395
396 def check_version(con):
397 try:
398 server_info = con.server_info()
399 except Exception as e:
400 return exit_with_general_critical(e), None
401 return 0, int(server_info['version'].split('.')[0].strip())
402
403 def check_connect(host, port, warning, critical, perf_data, user, passwd, conn_time):
404 warning = warning or 3
405 critical = critical or 6
406 message = "Connection took %.3f seconds" % conn_time
407 message += performance_data(perf_data, [(conn_time, "connection_time", warning, critical)])
408
409 return check_levels(conn_time, warning, critical, message)
410
411
412 def check_connections(con, warning, critical, perf_data):
413 warning = warning or 80
414 critical = critical or 95
415 try:
416 data = get_server_status(con)
417
418 current = float(data['connections']['current'])
419 available = float(data['connections']['available'])
420
421 used_percent = int(float(current / (available + current)) * 100)
422 message = "%i percent (%i of %i connections) used" % (used_percent, current, current + available)
423 message += performance_data(perf_data, [(used_percent, "used_percent", warning, critical),
424 (current, "current_connections"),
425 (available, "available_connections")])
426 return check_levels(used_percent, warning, critical, message)
427
428 except Exception as e:
429 return exit_with_general_critical(e)
430
431
432 def check_rep_lag(con, host, port, rdns_lookup, warning, critical, percent, perf_data, max_lag, ssl=False, user=None, passwd=None, replicaset=None, authdb="admin", insecure=None, ssl_ca_cert_file=None, cert_file=None, auth_mechanism=None, retry_writes_disabled=False):
433 # Get mongo to tell us replica set member name when connecting locally
434 if "127.0.0.1" == host:
435 if not "me" in list(con.admin.command("ismaster","1").keys()):
436 print("UNKNOWN - This is not replicated MongoDB")
437 return 3
438
439 host = con.admin.command("ismaster","1")["me"].split(':')[0]
440
441 if percent:
442 warning = warning or 50
443 critical = critical or 75
444 else:
445 warning = warning or 600
446 critical = critical or 3600
447 rs_status = {}
448 slaveDelays = {}
449 try:
450 #set_read_preference(con.admin)
451
452 # Get replica set status
453 try:
454 rs_status = con.admin.command("replSetGetStatus")
455 except pymongo.errors.OperationFailure as e:
456 if ((e.code == None and str(e).find('failed: not running with --replSet"')) or (e.code == 76 and str(e).find('not running with --replSet"'))):
457 print("UNKNOWN - Not running with replSet")
458 return 3
459 serverVersion = tuple(con.server_info()['version'].split('.'))
460 if serverVersion >= tuple("2.0.0".split(".")):
461 #
462 # check for version greater then 2.0
463 #
464 rs_conf = con.local.system.replset.find_one()
465 for member in rs_conf['members']:
466 if member.get('slaveDelay') is not None:
467 slaveDelays[member['host']] = member.get('slaveDelay')
468 else:
469 slaveDelays[member['host']] = 0
470
471 # Find the primary and/or the current node
472 primary_node = None
473 host_node = None
474
475 for member in rs_status["members"]:
476 if member["stateStr"] == "PRIMARY":
477 primary_node = member
478
479 # if rdns_lookup is true then lookup both values back to their rdns value so we can compare hostname vs fqdn
480 if rdns_lookup:
481 member_host, member_port = split_host_port(member.get('name'))
482 member_host = "{0}:{1}".format(socket.getnameinfo((member_host, 0), 0)[0], member_port)
483 if member_host == "{0}:{1}".format(socket.getnameinfo((host, 0), 0)[0], port):
484 host_node = member
485 # Exact match
486 elif member.get('name') == "{0}:{1}".format(host, port):
487 host_node = member
488
489 # Check if we're in the middle of an election and don't have a primary
490 if primary_node is None:
491 print("WARNING - No primary defined. In an election?")
492 return 1
493
494 # Check if we failed to find the current host
495 # below should never happen
496 if host_node is None:
497 print("CRITICAL - Unable to find host '" + host + "' in replica set.")
498 return 2
499
500 # Is the specified host the primary?
501 if host_node["stateStr"] == "PRIMARY":
502 if max_lag == False:
503 print("OK - This is the primary.")
504 return 0
505 else:
506 #get the maximal replication lag
507 data = ""
508 maximal_lag = 0
509 for member in rs_status['members']:
510 if not member['stateStr'] == "ARBITER":
511 lastSlaveOpTime = member['optimeDate']
512 replicationLag = abs(primary_node["optimeDate"] - lastSlaveOpTime).seconds - slaveDelays[member['name']]
513 data = data + member['name'] + " lag=%d;" % replicationLag
514 maximal_lag = max(maximal_lag, replicationLag)
515 if percent:
516 err, con = mongo_connect(split_host_port(primary_node['name'])[0], int(split_host_port(primary_node['name'])[1]), ssl, user, passwd, replicaset, authdb, insecure, ssl_ca_cert_file, cert_file, auth_mechanism, retry_writes_disabled=retry_writes_disabled)
517 if err != 0:
518 return err
519 primary_timediff = replication_get_time_diff(con)
520 maximal_lag = int(float(maximal_lag) / float(primary_timediff) * 100)
521 message = "Maximal lag is " + str(maximal_lag) + " percents"
522 message += performance_data(perf_data, [(maximal_lag, "replication_lag_percent", warning, critical)])
523 else:
524 message = "Maximal lag is " + str(maximal_lag) + " seconds"
525 message += performance_data(perf_data, [(maximal_lag, "replication_lag", warning, critical)])
526 return check_levels(maximal_lag, warning, critical, message)
527 elif host_node["stateStr"] == "ARBITER":
528 print("UNKNOWN - This is an arbiter")
529 return 3
530
531 # Find the difference in optime between current node and PRIMARY
532
533 optime_lag = abs(primary_node["optimeDate"] - host_node["optimeDate"])
534
535 if host_node['name'] in slaveDelays:
536 slave_delay = slaveDelays[host_node['name']]
537 elif host_node['name'].endswith(':27017') and host_node['name'][:-len(":27017")] in slaveDelays:
538 slave_delay = slaveDelays[host_node['name'][:-len(":27017")]]
539 else:
540 raise Exception("Unable to determine slave delay for {0}".format(host_node['name']))
541
542 try: # work starting from python2.7
543 lag = optime_lag.total_seconds()
544 except:
545 lag = float(optime_lag.seconds + optime_lag.days * 24 * 3600)
546
547 if percent:
548 err, con = mongo_connect(split_host_port(primary_node['name'])[0], int(split_host_port(primary_node['name'])[1]), ssl, user, passwd, replicaset, authdb, insecure, ssl_ca_cert_file, cert_file, auth_mechanism, retry_writes_disabled=retry_writes_disabled)
549 if err != 0:
550 return err
551 primary_timediff = replication_get_time_diff(con)
552 if primary_timediff != 0:
553 lag = int(float(lag) / float(primary_timediff) * 100)
554 else:
555 lag = 0
556 message = "Lag is " + str(lag) + " percents"
557 message += performance_data(perf_data, [(lag, "replication_lag_percent", warning, critical)])
558 else:
559 message = "Lag is " + str(lag) + " seconds"
560 message += performance_data(perf_data, [(lag, "replication_lag", warning, critical)])
561 return check_levels(lag, warning + slaveDelays[host_node['name']], critical + slaveDelays[host_node['name']], message)
562 else:
563 #
564 # less than 2.0 check
565 #
566 # Get replica set status
567 rs_status = con.admin.command("replSetGetStatus")
568
569 # Find the primary and/or the current node
570 primary_node = None
571 host_node = None
572 for member in rs_status["members"]:
573 if member["stateStr"] == "PRIMARY":
574 primary_node = (member["name"], member["optimeDate"])
575 if member["name"].split(":")[0].startswith(host):
576 host_node = member
577
578 # Check if we're in the middle of an election and don't have a primary
579 if primary_node is None:
580 print("WARNING - No primary defined. In an election?")
581 sys.exit(1)
582
583 # Is the specified host the primary?
584 if host_node["stateStr"] == "PRIMARY":
585 print("OK - This is the primary.")
586 sys.exit(0)
587
588 # Find the difference in optime between current node and PRIMARY
589 optime_lag = abs(primary_node[1] - host_node["optimeDate"])
590 lag = optime_lag.seconds
591 if percent:
592 err, con = mongo_connect(split_host_port(primary_node['name'])[0], int(split_host_port(primary_node['name'])[1]), ssl, user, passwd, replicaset, authdb, insecure, ssl_ca_cert_file, cert_file, auth_mechanism, retry_writes_disabled=retry_writes_disabled)
593 if err != 0:
594 return err
595 primary_timediff = replication_get_time_diff(con)
596 lag = int(float(lag) / float(primary_timediff) * 100)
597 message = "Lag is " + str(lag) + " percents"
598 message += performance_data(perf_data, [(lag, "replication_lag_percent", warning, critical)])
599 else:
600 message = "Lag is " + str(lag) + " seconds"
601 message += performance_data(perf_data, [(lag, "replication_lag", warning, critical)])
602 return check_levels(lag, warning, critical, message)
603
604 except Exception as e:
605 return exit_with_general_critical(e)
606
607 #
608 # Check the memory usage of mongo. Alerting on this may be hard to get right
609 # because it'll try to get as much memory as it can. And that's probably
610 # a good thing.
611 #
612 def check_memory(con, warning, critical, perf_data, mapped_memory, host):
613 # Get the total system memory of this system (This is totally bogus if you
614 # are running this command remotely) and calculate based on that how much
615 # memory used by Mongodb is ok or not.
616 meminfo = open('/proc/meminfo').read()
617 matched = re.search(r'^MemTotal:\s+(\d+)', meminfo)
618 if matched:
619 mem_total_kB = int(matched.groups()[0])
620
621 if host != "127.0.0.1" and not warning:
622 # Running remotely and value was not set by user, use hardcoded value
623 warning = 12
624 else:
625 # running locally or user provided value
626 warning = warning or (mem_total_kB * 0.8) / 1024.0 / 1024.0
627
628 if host != "127.0.0.1" and not critical:
629 critical = 16
630 else:
631 critical = critical or (mem_total_kB * 0.9) / 1024.0 / 1024.0
632
633 # debugging
634 #print "mem total: {0}kb, warn: {1}GB, crit: {2}GB".format(mem_total_kB,warning, critical)
635
636 try:
637 data = get_server_status(con)
638 if not data['mem']['supported'] and not mapped_memory:
639 print("OK - Platform not supported for memory info")
640 return 0
641 #
642 # convert to gigs
643 #
644 message = "Memory Usage:"
645 try:
646 mem_resident = float(data['mem']['resident']) / 1024.0
647 message += " %.2fGB resident," % (mem_resident)
648 except:
649 mem_resident = 0
650 message += " resident unsupported,"
651 try:
652 mem_virtual = float(data['mem']['virtual']) / 1024.0
653 message += " %.2fGB virtual," % mem_virtual
654 except:
655 mem_virtual = 0
656 message += " virtual unsupported,"
657 try:
658 mem_mapped = float(data['mem']['mapped']) / 1024.0
659 message += " %.2fGB mapped," % mem_mapped
660 except:
661 mem_mapped = 0
662 message += " mapped unsupported,"
663 try:
664 mem_mapped_journal = float(data['mem']['mappedWithJournal']) / 1024.0
665 message += " %.2fGB mappedWithJournal" % mem_mapped_journal
666 except:
667 mem_mapped_journal = 0
668 message += performance_data(perf_data, [("%.2f" % mem_resident, "memory_usage", warning, critical),
669 ("%.2f" % mem_mapped, "memory_mapped"), ("%.2f" % mem_virtual, "memory_virtual"), ("%.2f" % mem_mapped_journal, "mappedWithJournal")])
670 #added for unsupported systems like Solaris
671 if mapped_memory and mem_resident == 0:
672 return check_levels(mem_mapped, warning, critical, message)
673 else:
674 return check_levels(mem_resident, warning, critical, message)
675
676 except Exception as e:
677 return exit_with_general_critical(e)
678
679
680 def check_memory_mapped(con, warning, critical, perf_data):
681 #
682 # These thresholds are basically meaningless, and must be customized to your application
683 #
684 warning = warning or 8
685 critical = critical or 16
686 try:
687 data = get_server_status(con)
688 if not data['mem']['supported']:
689 print("OK - Platform not supported for memory info")
690 return 0
691 #
692 # convert to gigs
693 #
694 message = "Memory Usage:"
695 try:
696 mem_mapped = float(data['mem']['mapped']) / 1024.0
697 message += " %.2fGB mapped," % mem_mapped
698 except:
699 mem_mapped = -1
700 message += " mapped unsupported,"
701 try:
702 mem_mapped_journal = float(data['mem']['mappedWithJournal']) / 1024.0
703 message += " %.2fGB mappedWithJournal" % mem_mapped_journal
704 except:
705 mem_mapped_journal = 0
706 message += performance_data(perf_data, [("%.2f" % mem_mapped, "memory_mapped", warning, critical), ("%.2f" % mem_mapped_journal, "mappedWithJournal")])
707
708 if not mem_mapped == -1:
709 return check_levels(mem_mapped, warning, critical, message)
710 else:
711 print("OK - Server does not provide mem.mapped info")
712 return 0
713
714 except Exception as e:
715 return exit_with_general_critical(e)
716
717
718 #
719 # Return the percentage of the time there was a global Lock
720 #
721 def check_lock(con, warning, critical, perf_data, mongo_version):
722 warning = warning or 10
723 critical = critical or 30
724 if mongo_version == 2:
725 try:
726 data = get_server_status(con)
727 lockTime = data['globalLock']['lockTime']
728 totalTime = data['globalLock']['totalTime']
729 #
730 # calculate percentage
731 #
732 if lockTime > totalTime:
733 lock_percentage = 0.00
734 else:
735 lock_percentage = float(lockTime) / float(totalTime) * 100
736 message = "Lock Percentage: %.2f%%" % lock_percentage
737 message += performance_data(perf_data, [("%.2f" % lock_percentage, "lock_percentage", warning, critical)])
738 return check_levels(lock_percentage, warning, critical, message)
739 except Exception as e:
740 print("Couldn't get globalLock lockTime info from mongo, are you sure you're not using version 3? See the -M option.")
741 return exit_with_general_critical(e)
742 else:
743 print("OK - MongoDB version 3 doesn't report on global locks")
744 return 0
745
746
747 def check_flushing(con, warning, critical, avg, perf_data):
748 #
749 # These thresholds mean it's taking 5 seconds to perform a background flush to issue a warning
750 # and 10 seconds to issue a critical.
751 #
752 warning = warning or 5000
753 critical = critical or 15000
754 try:
755 data = get_server_status(con)
756 try:
757 data['backgroundFlushing']
758 if avg:
759 flush_time = float(data['backgroundFlushing']['average_ms'])
760 stat_type = "Average"
761 else:
762 flush_time = float(data['backgroundFlushing']['last_ms'])
763 stat_type = "Last"
764
765 message = "%s Flush Time: %.2fms" % (stat_type, flush_time)
766 message += performance_data(perf_data, [("%.2fms" % flush_time, "%s_flush_time" % stat_type.lower(), warning, critical)])
767
768 return check_levels(flush_time, warning, critical, message)
769 except Exception:
770 print("OK - flushing stats not available for this storage engine")
771 return 0
772
773 except Exception as e:
774 return exit_with_general_critical(e)
775
776
777 def index_miss_ratio(con, warning, critical, perf_data):
778 warning = warning or 10
779 critical = critical or 30
780 try:
781 data = get_server_status(con)
782
783 try:
784 data['indexCounters']
785 serverVersion = tuple(con.server_info()['version'].split('.'))
786 if serverVersion >= tuple("2.4.0".split(".")):
787 miss_ratio = float(data['indexCounters']['missRatio'])
788 else:
789 miss_ratio = float(data['indexCounters']['btree']['missRatio'])
790 except KeyError:
791 not_supported_msg = "not supported on this platform"
792 try:
793 data['indexCounters']
794 if 'note' in data['indexCounters']:
795 print("OK - MongoDB says: " + not_supported_msg)
796 return 0
797 else:
798 print("WARNING - Can't get counter from MongoDB")
799 return 1
800 except Exception:
801 print("OK - MongoDB says: " + not_supported_msg)
802 return 0
803
804 message = "Miss Ratio: %.2f" % miss_ratio
805 message += performance_data(perf_data, [("%.2f" % miss_ratio, "index_miss_ratio", warning, critical)])
806
807 return check_levels(miss_ratio, warning, critical, message)
808
809 except Exception as e:
810 return exit_with_general_critical(e)
811
812 def check_replset_quorum(con, perf_data):
813 db = con['admin']
814 warning = 1
815 critical = 2
816 primary = 0
817
818 try:
819 rs_members = db.command("replSetGetStatus")['members']
820
821 for member in rs_members:
822 if member['state'] == 1:
823 primary += 1
824
825 if primary == 1:
826 state = 0
827 message = "Cluster is quorate"
828 else:
829 state = 2
830 message = "Cluster is not quorate and cannot operate"
831
832 return check_levels(state, warning, critical, message)
833 except Exception as e:
834 return exit_with_general_critical(e)
835
836
837
838 def check_replset_state(con, perf_data, warning="", critical=""):
839 try:
840 warning = [int(x) for x in warning.split(",")]
841 except:
842 warning = [0, 3, 5]
843 try:
844 critical = [int(x) for x in critical.split(",")]
845 except:
846 critical = [8, 4, -1]
847
848 ok = list(range(-1, 8)) # should include the range of all posiible values
849 try:
850 worst_state = -2
851 message = ""
852 try:
853 try:
854 set_read_preference(con.admin)
855 data = con.admin.command(pymongo.son_manipulator.SON([('replSetGetStatus', 1)]))
856 except:
857 data = con.admin.command(son.SON([('replSetGetStatus', 1)]))
858 members = data['members']
859 my_state = int(data['myState'])
860 worst_state = my_state
861 for member in members:
862 their_state = int(member['state'])
863 message += " %s: %i (%s)" % (member['name'], their_state, state_text(their_state))
864 if state_is_worse(their_state, worst_state, warning, critical):
865 worst_state = their_state
866 message += performance_data(perf_data, [(my_state, "state")])
867
868 except pymongo.errors.OperationFailure as e:
869 if ((e.code == None and str(e).find('failed: not running with --replSet"')) or (e.code == 76 and str(e).find('not running with --replSet"'))):
870 worst_state = -1
871
872 return check_levels(worst_state, warning, critical, message, ok)
873 except Exception as e:
874 return exit_with_general_critical(e)
875
876 def state_is_worse(state, worst_state, warning, critical):
877 if worst_state in critical:
878 return False
879 if worst_state in warning:
880 return state in critical
881 return (state in warning) or (state in critical)
882
883 def state_text(state):
884 if state == 8:
885 return "Down"
886 elif state == 4:
887 return "Fatal error"
888 elif state == 0:
889 return "Starting up, phase1"
890 elif state == 3:
891 return "Recovering"
892 elif state == 5:
893 return "Starting up, phase2"
894 elif state == 1:
895 return "Primary"
896 elif state == 2:
897 return "Secondary"
898 elif state == 7:
899 return "Arbiter"
900 elif state == -1:
901 return "Not running with replSet"
902 else:
903 return "Unknown state"
904
905
906 def check_databases(con, warning, critical, perf_data=None):
907 try:
908 try:
909 set_read_preference(con.admin)
910 data = con.admin.command(pymongo.son_manipulator.SON([('listDatabases', 1)]))
911 except:
912 data = con.admin.command(son.SON([('listDatabases', 1)]))
913
914 count = len(data['databases'])
915 message = "Number of DBs: %.0f" % count
916 message += performance_data(perf_data, [(count, "databases", warning, critical, message)])
917 return check_levels(count, warning, critical, message)
918 except Exception as e:
919 return exit_with_general_critical(e)
920
921
922 def check_collections(con, warning, critical, perf_data=None):
923 try:
924 try:
925 set_read_preference(con.admin)
926 data = con.admin.command(pymongo.son_manipulator.SON([('listDatabases', 1)]))
927 except:
928 data = con.admin.command(son.SON([('listDatabases', 1)]))
929
930 count = 0
931 for db in data['databases']:
932 dbase = con[db['name']]
933 set_read_preference(dbase)
934 count += len(dbase.collection_names())
935
936 message = "Number of collections: %.0f" % count
937 message += performance_data(perf_data, [(count, "collections", warning, critical, message)])
938 return check_levels(count, warning, critical, message)
939
940 except Exception as e:
941 return exit_with_general_critical(e)
942
943
944 def check_all_databases_size(con, warning, critical, perf_data):
945 warning = warning or 100
946 critical = critical or 1000
947 try:
948 set_read_preference(con.admin)
949 all_dbs_data = con.admin.command(pymongo.son_manipulator.SON([('listDatabases', 1)]))
950 except:
951 all_dbs_data = con.admin.command(son.SON([('listDatabases', 1)]))
952
953 total_storage_size = 0
954 message = ""
955 perf_data_param = [()]
956 for db in all_dbs_data['databases']:
957 database = db['name']
958 data = con[database].command('dbstats')
959 storage_size = round(data['storageSize'] / 1024 / 1024, 1)
960 message += "; Database %s size: %.0f MB" % (database, storage_size)
961 perf_data_param.append((storage_size, database + "_database_size"))
962 total_storage_size += storage_size
963
964 perf_data_param[0] = (total_storage_size, "total_size", warning, critical)
965 message += performance_data(perf_data, perf_data_param)
966 message = "Total size: %.0f MB" % total_storage_size + message
967 return check_levels(total_storage_size, warning, critical, message)
968
969
970 def check_database_size(con, database, warning, critical, perf_data):
971 warning = warning or 100
972 critical = critical or 1000
973 perfdata = ""
974 try:
975 set_read_preference(con.admin)
976 data = con[database].command('dbstats')
977 storage_size = data['storageSize'] // 1024 // 1024
978 if perf_data:
979 perfdata += " | database_size=%i;%i;%i" % (storage_size, warning, critical)
980 #perfdata += " database=%s" %(database)
981
982 if storage_size >= critical:
983 print("CRITICAL - Database size: %.0f MB, Database: %s%s" % (storage_size, database, perfdata))
984 return 2
985 elif storage_size >= warning:
986 print("WARNING - Database size: %.0f MB, Database: %s%s" % (storage_size, database, perfdata))
987 return 1
988 else:
989 print("OK - Database size: %.0f MB, Database: %s%s" % (storage_size, database, perfdata))
990 return 0
991 except Exception as e:
992 return exit_with_general_critical(e)
993
994
995 def check_database_indexes(con, database, warning, critical, perf_data):
996 #
997 # These thresholds are basically meaningless, and must be customized to your application
998 #
999 warning = warning or 100
1000 critical = critical or 1000
1001 perfdata = ""
1002 try:
1003 set_read_preference(con.admin)
1004 data = con[database].command('dbstats')
1005 index_size = data['indexSize'] / 1024 // 1024
1006 if perf_data:
1007 perfdata += " | database_indexes=%i;%i;%i" % (index_size, warning, critical)
1008
1009 if index_size >= critical:
1010 print("CRITICAL - %s indexSize: %.0f MB %s" % (database, index_size, perfdata))
1011 return 2
1012 elif index_size >= warning:
1013 print("WARNING - %s indexSize: %.0f MB %s" % (database, index_size, perfdata))
1014 return 1
1015 else:
1016 print("OK - %s indexSize: %.0f MB %s" % (database, index_size, perfdata))
1017 return 0
1018 except Exception as e:
1019 return exit_with_general_critical(e)
1020
1021
1022 def check_collection_documents(con, database, collection, warning, critical, perf_data):
1023 perfdata = ""
1024 try:
1025 set_read_preference(con.admin)
1026 data = con[database].command('collstats', collection)
1027 documents = data['count']
1028 if perf_data:
1029 perfdata += " | collection_documents=%i;%i;%i" % (documents, warning, critical)
1030
1031 if documents >= critical:
1032 print("CRITICAL - %s.%s documents: %s %s" % (database, collection, documents, perfdata))
1033 return 2
1034 elif documents >= warning:
1035 print("WARNING - %s.%s documents: %s %s" % (database, collection, documents, perfdata))
1036 return 1
1037 else:
1038 print("OK - %s.%s documents: %s %s" % (database, collection, documents, perfdata))
1039 return 0
1040 except Exception as e:
1041 return exit_with_general_critical(e)
1042
1043
1044 def check_collection_indexes(con, database, collection, warning, critical, perf_data):
1045 #
1046 # These thresholds are basically meaningless, and must be customized to your application
1047 #
1048 warning = warning or 100
1049 critical = critical or 1000
1050 perfdata = ""
1051 try:
1052 set_read_preference(con.admin)
1053 data = con[database].command('collstats', collection)
1054 total_index_size = data['totalIndexSize'] / 1024 / 1024
1055 if perf_data:
1056 perfdata += " | collection_indexes=%i;%i;%i" % (total_index_size, warning, critical)
1057
1058 if total_index_size >= critical:
1059 print("CRITICAL - %s.%s totalIndexSize: %.0f MB %s" % (database, collection, total_index_size, perfdata))
1060 return 2
1061 elif total_index_size >= warning:
1062 print("WARNING - %s.%s totalIndexSize: %.0f MB %s" % (database, collection, total_index_size, perfdata))
1063 return 1
1064 else:
1065 print("OK - %s.%s totalIndexSize: %.0f MB %s" % (database, collection, total_index_size, perfdata))
1066 return 0
1067 except Exception as e:
1068 return exit_with_general_critical(e)
1069
1070
1071 def check_queues(con, warning, critical, perf_data):
1072 warning = warning or 10
1073 critical = critical or 30
1074 try:
1075 data = get_server_status(con)
1076
1077 total_queues = float(data['globalLock']['currentQueue']['total'])
1078 readers_queues = float(data['globalLock']['currentQueue']['readers'])
1079 writers_queues = float(data['globalLock']['currentQueue']['writers'])
1080 message = "Current queue is : total = %d, readers = %d, writers = %d" % (total_queues, readers_queues, writers_queues)
1081 message += performance_data(perf_data, [(total_queues, "total_queues", warning, critical), (readers_queues, "readers_queues"), (writers_queues, "writers_queues")])
1082 return check_levels(total_queues, warning, critical, message)
1083
1084 except Exception as e:
1085 return exit_with_general_critical(e)
1086
1087 def check_collection_size(con, database, collection, warning, critical, perf_data):
1088 warning = warning or 100
1089 critical = critical or 1000
1090 perfdata = ""
1091 try:
1092 set_read_preference(con.admin)
1093 data = con[database].command('collstats', collection)
1094 size = data['size'] / 1024 / 1024
1095 if perf_data:
1096 perfdata += " | collection_size=%i;%i;%i" % (size, warning, critical)
1097
1098 if size >= critical:
1099 print("CRITICAL - %s.%s size: %.0f MB %s" % (database, collection, size, perfdata))
1100 return 2
1101 elif size >= warning:
1102 print("WARNING - %s.%s size: %.0f MB %s" % (database, collection, size, perfdata))
1103 return 1
1104 else:
1105 print("OK - %s.%s size: %.0f MB %s" % (database, collection, size, perfdata))
1106 return 0
1107 except Exception as e:
1108 return exit_with_general_critical(e)
1109
1110
1111 def check_collection_storageSize(con, database, collection, warning, critical, perf_data):
1112 warning = warning or 100
1113 critical = critical or 1000
1114 perfdata = ""
1115 try:
1116 set_read_preference(con.admin)
1117 data = con[database].command('collstats', collection)
1118 storageSize = data['storageSize'] / 1024 / 1024
1119 if perf_data:
1120 perfdata += " | collection_storageSize=%i;%i;%i" % (storageSize, warning, critical)
1121
1122 if storageSize >= critical:
1123 print("CRITICAL - %s.%s storageSize: %.0f MB %s" % (database, collection, storageSize, perfdata))
1124 return 2
1125 elif storageSize >= warning:
1126 print("WARNING - %s.%s storageSize: %.0f MB %s" % (database, collection, storageSize, perfdata))
1127 return 1
1128 else:
1129 print("OK - %s.%s storageSize: %.0f MB %s" % (database, collection, storageSize, perfdata))
1130 return 0
1131 except Exception as e:
1132 return exit_with_general_critical(e)
1133
1134
1135 def check_queries_per_second(con, query_type, warning, critical, perf_data, mongo_version):
1136 warning = warning or 250
1137 critical = critical or 500
1138
1139 if query_type not in ['insert', 'query', 'update', 'delete', 'getmore', 'command']:
1140 return exit_with_general_critical("The query type of '%s' is not valid" % query_type)
1141
1142 try:
1143 db = con.local
1144 data = get_server_status(con)
1145
1146 # grab the count
1147 num = int(data['opcounters'][query_type])
1148
1149 # do the math
1150 last_count = db.nagios_check.find_one({'check': 'query_counts'})
1151 try:
1152 ts = int(time.time())
1153 diff_query = num - last_count['data'][query_type]['count']
1154 diff_ts = ts - last_count['data'][query_type]['ts']
1155
1156 if diff_ts == 0:
1157 message = "diff_query = " + str(diff_query) + " diff_ts = " + str(diff_ts)
1158 return check_levels(0, warning, critical, message)
1159
1160 query_per_sec = float(diff_query) / float(diff_ts)
1161
1162 # update the count now
1163 if mongo_version == 2:
1164 db.nagios_check.update({u'_id': last_count['_id']}, {'$set': {"data.%s" % query_type: {'count': num, 'ts': int(time.time())}}})
1165 else:
1166 db.nagios_check.update_one({u'_id': last_count['_id']}, {'$set': {"data.%s" % query_type: {'count': num, 'ts': int(time.time())}}})
1167
1168 message = "Queries / Sec: %f" % query_per_sec
1169 message += performance_data(perf_data, [(query_per_sec, "%s_per_sec" % query_type, warning, critical, message)])
1170 except KeyError:
1171 #
1172 # since it is the first run insert it
1173 query_per_sec = 0
1174 message = "First run of check.. no data"
1175 if mongo_version == 2:
1176 db.nagios_check.update({u'_id': last_count['_id']}, {'$set': {"data.%s" % query_type: {'count': num, 'ts': int(time.time())}}})
1177 else:
1178 db.nagios_check.update_one({u'_id': last_count['_id']}, {'$set': {"data.%s" % query_type: {'count': num, 'ts': int(time.time())}}})
1179
1180 except TypeError:
1181 #
1182 # since it is the first run insert it
1183 query_per_sec = 0
1184 message = "First run of check.. no data"
1185 if mongo_version == 2:
1186 db.nagios_check.insert({'check': 'query_counts', 'data': {query_type: {'count': num, 'ts': int(time.time())}}})
1187 else:
1188 db.nagios_check.insert_one({'check': 'query_counts', 'data': {query_type: {'count': num, 'ts': int(time.time())}}})
1189
1190 return check_levels(query_per_sec, warning, critical, message)
1191
1192 except Exception as e:
1193 return exit_with_general_critical(e)
1194
1195
1196 def check_oplog(con, warning, critical, perf_data):
1197 """ Checking the oplog time - the time of the log currntly saved in the oplog collection
1198 defaults:
1199 critical 4 hours
1200 warning 24 hours
1201 those can be changed as usual with -C and -W parameters"""
1202 warning = warning or 24
1203 critical = critical or 4
1204 try:
1205 db = con.local
1206 ol = db.system.namespaces.find_one({"name": "local.oplog.rs"})
1207 if (db.system.namespaces.find_one({"name": "local.oplog.rs"}) != None):
1208 oplog = "oplog.rs"
1209 else:
1210 ol = db.system.namespaces.find_one({"name": "local.oplog.$main"})
1211 if (db.system.namespaces.find_one({"name": "local.oplog.$main"}) != None):
1212 oplog = "oplog.$main"
1213 else:
1214 message = "neither master/slave nor replica set replication detected"
1215 return check_levels(None, warning, critical, message)
1216
1217 try:
1218 set_read_preference(con.admin)
1219 data = con.local.command(pymongo.son_manipulator.SON([('collstats', oplog)]))
1220 except:
1221 data = con.admin.command(son.SON([('collstats', oplog)]))
1222
1223 ol_size = data['size']
1224 ol_storage_size = data['storageSize']
1225 ol_used_storage = int(float(ol_size) / ol_storage_size * 100 + 1)
1226 ol = con.local[oplog]
1227 firstc = ol.find().sort("$natural", pymongo.ASCENDING).limit(1)[0]['ts']
1228 lastc = ol.find().sort("$natural", pymongo.DESCENDING).limit(1)[0]['ts']
1229 time_in_oplog = (lastc.as_datetime() - firstc.as_datetime())
1230 message = "Oplog saves " + str(time_in_oplog) + " %d%% used" % ol_used_storage
1231 try: # work starting from python2.7
1232 hours_in_oplog = time_in_oplog.total_seconds() / 60 / 60
1233 except:
1234 hours_in_oplog = float(time_in_oplog.seconds + time_in_oplog.days * 24 * 3600) / 60 / 60
1235 approx_level = hours_in_oplog * 100 / ol_used_storage
1236 message += performance_data(perf_data, [("%.2f" % hours_in_oplog, 'oplog_time', warning, critical), ("%.2f " % approx_level, 'oplog_time_100_percent_used')])
1237 return check_levels(-approx_level, -warning, -critical, message)
1238
1239 except Exception as e:
1240 return exit_with_general_critical(e)
1241
1242
1243 def check_journal_commits_in_wl(con, warning, critical, perf_data):
1244 """ Checking the number of commits which occurred in the db's write lock.
1245 Most commits are performed outside of this lock; committed while in the write lock is undesirable.
1246 Under very high write situations it is normal for this value to be nonzero. """
1247
1248 warning = warning or 10
1249 critical = critical or 40
1250 try:
1251 data = get_server_status(con)
1252 j_commits_in_wl = data['dur']['commitsInWriteLock']
1253 message = "Journal commits in DB write lock : %d" % j_commits_in_wl
1254 message += performance_data(perf_data, [(j_commits_in_wl, "j_commits_in_wl", warning, critical)])
1255 return check_levels(j_commits_in_wl, warning, critical, message)
1256
1257 except Exception as e:
1258 return exit_with_general_critical(e)
1259
1260
1261 def check_journaled(con, warning, critical, perf_data):
1262 """ Checking the average amount of data in megabytes written to the recovery log in the last four seconds"""
1263
1264 warning = warning or 20
1265 critical = critical or 40
1266 try:
1267 data = get_server_status(con)
1268 journaled = data['dur']['journaledMB']
1269 message = "Journaled : %.2f MB" % journaled
1270 message += performance_data(perf_data, [("%.2f" % journaled, "journaled", warning, critical)])
1271 return check_levels(journaled, warning, critical, message)
1272
1273 except Exception as e:
1274 return exit_with_general_critical(e)
1275
1276
1277 def check_write_to_datafiles(con, warning, critical, perf_data):
1278 """ Checking the average amount of data in megabytes written to the databases datafiles in the last four seconds.
1279 As these writes are already journaled, they can occur lazily, and thus the number indicated here may be lower
1280 than the amount physically written to disk."""
1281 warning = warning or 20
1282 critical = critical or 40
1283 try:
1284 data = get_server_status(con)
1285 writes = data['dur']['writeToDataFilesMB']
1286 message = "Write to data files : %.2f MB" % writes
1287 message += performance_data(perf_data, [("%.2f" % writes, "write_to_data_files", warning, critical)])
1288 return check_levels(writes, warning, critical, message)
1289
1290 except Exception as e:
1291 return exit_with_general_critical(e)
1292
1293
1294 def get_opcounters(data, opcounters_name, host, port):
1295 try:
1296 insert = data[opcounters_name]['insert']
1297 query = data[opcounters_name]['query']
1298 update = data[opcounters_name]['update']
1299 delete = data[opcounters_name]['delete']
1300 getmore = data[opcounters_name]['getmore']
1301 command = data[opcounters_name]['command']
1302 except KeyError as e:
1303 return 0, [0] * 100
1304 total_commands = insert + query + update + delete + getmore + command
1305 new_vals = [total_commands, insert, query, update, delete, getmore, command]
1306 return maintain_delta(new_vals, host, port, opcounters_name)
1307
1308
1309 def check_opcounters(con, host, port, warning, critical, perf_data):
1310 """ A function to get all opcounters delta per minute. In case of a replication - gets the opcounters+opcountersRepl"""
1311 warning = warning or 10000
1312 critical = critical or 15000
1313
1314 data = get_server_status(con)
1315 err1, delta_opcounters = get_opcounters(data, 'opcounters', host, port)
1316 err2, delta_opcounters_repl = get_opcounters(data, 'opcountersRepl', host, port)
1317 if err1 == 0 and err2 == 0:
1318 delta = [(x + y) for x, y in zip(delta_opcounters, delta_opcounters_repl)]
1319 delta[0] = delta_opcounters[0] # only the time delta shouldn't be summarized
1320 per_minute_delta = [int(x / delta[0] * 60) for x in delta[1:]]
1321 message = "Test succeeded , old values missing"
1322 message = "Opcounters: total=%d,insert=%d,query=%d,update=%d,delete=%d,getmore=%d,command=%d" % tuple(per_minute_delta)
1323 message += performance_data(perf_data, ([(per_minute_delta[0], "total", warning, critical), (per_minute_delta[1], "insert"),
1324 (per_minute_delta[2], "query"), (per_minute_delta[3], "update"), (per_minute_delta[4], "delete"),
1325 (per_minute_delta[5], "getmore"), (per_minute_delta[6], "command")]))
1326 return check_levels(per_minute_delta[0], warning, critical, message)
1327 else:
1328 return exit_with_general_critical("problem reading data from temp file")
1329
1330
1331 def check_current_lock(con, host, port, warning, critical, perf_data):
1332 """ A function to get current lock percentage and not a global one, as check_lock function does"""
1333 warning = warning or 10
1334 critical = critical or 30
1335 data = get_server_status(con)
1336
1337 lockTime = float(data['globalLock']['lockTime'])
1338 totalTime = float(data['globalLock']['totalTime'])
1339
1340 err, delta = maintain_delta([totalTime, lockTime], host, port, "locktime")
1341 if err == 0:
1342 lock_percentage = delta[2] / delta[1] * 100 # lockTime/totalTime*100
1343 message = "Current Lock Percentage: %.2f%%" % lock_percentage
1344 message += performance_data(perf_data, [("%.2f" % lock_percentage, "current_lock_percentage", warning, critical)])
1345 return check_levels(lock_percentage, warning, critical, message)
1346 else:
1347 return exit_with_general_warning("problem reading data from temp file")
1348
1349
1350 def check_page_faults(con, host, port, warning, critical, perf_data):
1351 """ A function to get page_faults per second from the system"""
1352 warning = warning or 10
1353 critical = critical or 30
1354 data = get_server_status(con)
1355
1356 try:
1357 page_faults = float(data['extra_info']['page_faults'])
1358 except:
1359 # page_faults unsupported on the underlaying system
1360 return exit_with_general_critical("page_faults unsupported on the underlaying system")
1361
1362 err, delta = maintain_delta([page_faults], host, port, "page_faults")
1363 if err == 0:
1364 page_faults_ps = delta[1] / delta[0]
1365 message = "Page faults : %.2f ps" % page_faults_ps
1366 message += performance_data(perf_data, [("%.2f" % page_faults_ps, "page_faults_ps", warning, critical)])
1367 return check_levels(page_faults_ps, warning, critical, message)
1368 else:
1369 return exit_with_general_warning("problem reading data from temp file")
1370
1371
1372 def check_asserts(con, host, port, warning, critical, perf_data):
1373 """ A function to get asserts from the system"""
1374 warning = warning or 1
1375 critical = critical or 10
1376 data = get_server_status(con)
1377
1378 asserts = data['asserts']
1379
1380 #{ "regular" : 0, "warning" : 6, "msg" : 0, "user" : 12, "rollovers" : 0 }
1381 regular = asserts['regular']
1382 warning_asserts = asserts['warning']
1383 msg = asserts['msg']
1384 user = asserts['user']
1385 rollovers = asserts['rollovers']
1386
1387 err, delta = maintain_delta([regular, warning_asserts, msg, user, rollovers], host, port, "asserts")
1388
1389 if err == 0:
1390 if delta[5] != 0:
1391 #the number of rollovers were increased
1392 warning = -1 # no matter the metrics this situation should raise a warning
1393 # if this is normal rollover - the warning will not appear again, but if there will be a lot of asserts
1394 # the warning will stay for a long period of time
1395 # although this is not a usual situation
1396
1397 regular_ps = delta[1] / delta[0]
1398 warning_ps = delta[2] / delta[0]
1399 msg_ps = delta[3] / delta[0]
1400 user_ps = delta[4] / delta[0]
1401 rollovers_ps = delta[5] / delta[0]
1402 total_ps = regular_ps + warning_ps + msg_ps + user_ps
1403 message = "Total asserts : %.2f ps" % total_ps
1404 message += performance_data(perf_data, [(total_ps, "asserts_ps", warning, critical), (regular_ps, "regular"),
1405 (warning_ps, "warning"), (msg_ps, "msg"), (user_ps, "user")])
1406 return check_levels(total_ps, warning, critical, message)
1407 else:
1408 return exit_with_general_warning("problem reading data from temp file")
1409
1410
1411 def get_stored_primary_server_name(db):
1412 """ get the stored primary server name from db. """
1413 if "last_primary_server" in db.collection_names():
1414 stored_primary_server = db.last_primary_server.find_one()["server"]
1415 else:
1416 stored_primary_server = None
1417
1418 return stored_primary_server
1419
1420
1421 def check_replica_primary(con, host, warning, critical, perf_data, replicaset, mongo_version):
1422 """ A function to check if the primary server of a replica set has changed """
1423 if warning is None and critical is None:
1424 warning = 1
1425 warning = warning or 2
1426 critical = critical or 2
1427
1428 primary_status = 0
1429 message = "Primary server has not changed"
1430 db = con["nagios"]
1431 data = get_server_status(con)
1432 if replicaset != data['repl'].get('setName'):
1433 message = "Replica set requested: %s differs from the one found: %s" % (replicaset, data['repl'].get('setName'))
1434 primary_status = 2
1435 return check_levels(primary_status, warning, critical, message)
1436 current_primary = data['repl'].get('primary')
1437 saved_primary = get_stored_primary_server_name(db)
1438 if current_primary is None:
1439 current_primary = "None"
1440 if saved_primary is None:
1441 saved_primary = "None"
1442 if current_primary != saved_primary:
1443 last_primary_server_record = {"server": current_primary}
1444 if mongo_version == 2:
1445 db.last_primary_server.update({"_id": "last_primary"}, {"$set": last_primary_server_record}, upsert=True)
1446 else:
1447 db.last_primary_server.update_one({"_id": "last_primary"}, {"$set": last_primary_server_record}, upsert=True)
1448 message = "Primary server has changed from %s to %s" % (saved_primary, current_primary)
1449 primary_status = 1
1450 return check_levels(primary_status, warning, critical, message)
1451
1452
1453 def check_page_faults(con, sample_time, warning, critical, perf_data):
1454 warning = warning or 10
1455 critical = critical or 20
1456 try:
1457 try:
1458 set_read_preference(con.admin)
1459 data1 = con.admin.command(pymongo.son_manipulator.SON([('serverStatus', 1)]))
1460 time.sleep(sample_time)
1461 data2 = con.admin.command(pymongo.son_manipulator.SON([('serverStatus', 1)]))
1462 except:
1463 data1 = con.admin.command(son.SON([('serverStatus', 1)]))
1464 time.sleep(sample_time)
1465 data2 = con.admin.command(son.SON([('serverStatus', 1)]))
1466
1467 try:
1468 #on linux servers only
1469 page_faults = (int(data2['extra_info']['page_faults']) - int(data1['extra_info']['page_faults'])) // sample_time
1470 except KeyError:
1471 print("WARNING - Can't get extra_info.page_faults counter from MongoDB")
1472 sys.exit(1)
1473
1474 message = "Page Faults: %i" % (page_faults)
1475
1476 message += performance_data(perf_data, [(page_faults, "page_faults", warning, critical)])
1477 check_levels(page_faults, warning, critical, message)
1478
1479 except Exception as e:
1480 exit_with_general_critical(e)
1481
1482
1483 def chunks_balance(con, database, collection, warning, critical):
1484 warning = warning or 10
1485 critical = critical or 20
1486 nsfilter = database + "." + collection
1487 try:
1488 try:
1489 set_read_preference(con.admin)
1490 col = con.config.chunks
1491 nscount = col.find({"ns": nsfilter}).count()
1492 shards = col.distinct("shard")
1493
1494 except:
1495 print("WARNING - Can't get chunks infos from MongoDB")
1496 sys.exit(1)
1497
1498 if nscount == 0:
1499 print("WARNING - Namespace %s is not sharded" % (nsfilter))
1500 sys.exit(1)
1501
1502 avgchunksnb = nscount // len(shards)
1503 warningnb = avgchunksnb * warning // 100
1504 criticalnb = avgchunksnb * critical // 100
1505
1506 for shard in shards:
1507 delta = abs(avgchunksnb - col.find({"ns": nsfilter, "shard": shard}).count())
1508 message = "Namespace: %s, Shard name: %s, Chunk delta: %i" % (nsfilter, shard, delta)
1509
1510 if delta >= criticalnb and delta > 0:
1511 print("CRITICAL - Chunks not well balanced " + message)
1512 sys.exit(2)
1513 elif delta >= warningnb and delta > 0:
1514 print("WARNING - Chunks not well balanced " + message)
1515 sys.exit(1)
1516
1517 print("OK - Chunks well balanced across shards")
1518 sys.exit(0)
1519
1520 except Exception as e:
1521 exit_with_general_critical(e)
1522
1523 print("OK - Chunks well balanced across shards")
1524 sys.exit(0)
1525
1526
1527 def check_connect_primary(con, warning, critical, perf_data):
1528 warning = warning or 3
1529 critical = critical or 6
1530
1531 try:
1532 try:
1533 set_read_preference(con.admin)
1534 data = con.admin.command(pymongo.son_manipulator.SON([('isMaster', 1)]))
1535 except:
1536 data = con.admin.command(son.SON([('isMaster', 1)]))
1537
1538 if data['ismaster'] == True:
1539 print("OK - This server is primary")
1540 return 0
1541
1542 phost = data['primary'].split(':')[0]
1543 pport = int(data['primary'].split(':')[1])
1544 start = time.time()
1545
1546 err, con = mongo_connect(phost, pport)
1547 if err != 0:
1548 return err
1549
1550 pconn_time = time.time() - start
1551 pconn_time = round(pconn_time, 0)
1552 message = "Connection to primary server " + data['primary'] + " took %i seconds" % pconn_time
1553 message += performance_data(perf_data, [(pconn_time, "connection_time", warning, critical)])
1554
1555 return check_levels(pconn_time, warning, critical, message)
1556
1557 except Exception as e:
1558 return exit_with_general_critical(e)
1559
1560
1561 def check_collection_state(con, database, collection):
1562 try:
1563 con[database][collection].find_one()
1564 print("OK - Collection %s.%s is reachable " % (database, collection))
1565 return 0
1566
1567 except Exception as e:
1568 return exit_with_general_critical(e)
1569
1570
1571 def check_row_count(con, database, collection, warning, critical, perf_data):
1572 try:
1573 count = con[database][collection].count()
1574 message = "Row count: %i" % (count)
1575 message += performance_data(perf_data, [(count, "row_count", warning, critical)])
1576
1577 return check_levels(count, warning, critical, message)
1578
1579 except Exception as e:
1580 return exit_with_general_critical(e)
1581
1582
1583 def build_file_name(host, port, action):
1584 #done this way so it will work when run independently and from shell
1585 module_name = re.match('(.*//*)*(.*)\..*', __file__).group(2)
1586
1587 if (port == 27017):
1588 return "/tmp/" + module_name + "_data/" + host + "-" + action + ".data"
1589 else:
1590 return "/tmp/" + module_name + "_data/" + host + "-" + str(port) + "-" + action + ".data"
1591
1592
1593 def ensure_dir(f):
1594 d = os.path.dirname(f)
1595 if not os.path.exists(d):
1596 os.makedirs(d)
1597
1598
1599 def write_values(file_name, string):
1600 f = None
1601 try:
1602 f = open(file_name, 'w')
1603 except IOError as e:
1604 #try creating
1605 if (e.errno == 2):
1606 ensure_dir(file_name)
1607 f = open(file_name, 'w')
1608 else:
1609 raise IOError(e)
1610 f.write(string)
1611 f.close()
1612 return 0
1613
1614
1615 def read_values(file_name):
1616 data = None
1617 try:
1618 f = open(file_name, 'r')
1619 data = f.read()
1620 f.close()
1621 return 0, data
1622 except IOError as e:
1623 if (e.errno == 2):
1624 #no previous data
1625 return 1, ''
1626 except Exception as e:
1627 return 2, None
1628
1629
1630 def calc_delta(old, new):
1631 delta = []
1632 if (len(old) != len(new)):
1633 raise Exception("unequal number of parameters")
1634 for i in range(0, len(old)):
1635 val = float(new[i]) - float(old[i])
1636 if val < 0:
1637 val = new[i]
1638 delta.append(val)
1639 return 0, delta
1640
1641
1642 def maintain_delta(new_vals, host, port, action):
1643 file_name = build_file_name(host, port, action)
1644 err, data = read_values(file_name)
1645 old_vals = data.split(';')
1646 new_vals = [str(int(time.time()))] + new_vals
1647 delta = None
1648 try:
1649 err, delta = calc_delta(old_vals, new_vals)
1650 except:
1651 err = 2
1652 write_res = write_values(file_name, ";" . join(str(x) for x in new_vals))
1653 return err + write_res, delta
1654
1655
1656 def replication_get_time_diff(con):
1657 col = 'oplog.rs'
1658 local = con.local
1659 ol = local.system.namespaces.find_one({"name": "local.oplog.$main"})
1660 if ol:
1661 col = 'oplog.$main'
1662 firstc = local[col].find().sort("$natural", 1).limit(1)
1663 lastc = local[col].find().sort("$natural", -1).limit(1)
1664 first = next(firstc)
1665 last = next(lastc)
1666 tfirst = first["ts"]
1667 tlast = last["ts"]
1668 delta = tlast.time - tfirst.time
1669 return delta
1670
1671 #
1672 # main app
1673 #
1674 if __name__ == "__main__":
1675 sys.exit(main(sys.argv[1:]))