Debian Codegraph @

scripts/kprobes_test/monitor_system.py

raw
1 #!/usr/bin/python
2
3 # Copyright (C) 2008, 2014 Red Hat Inc.
4 #
5 # This file is part of systemtap, and is free software. You can
6 # redistribute it and/or modify it under the terms of the GNU General
7 # Public License (GPL); either version 2, or (at your option) any
8 # later version.
9
10 # This script monitors a remote system that is running the kprobes
11 # test. If several consecutive 'ping's fail, the system is rebooted.
12 #
13 # This script takes as an argument a config filename, whose contents
14 # should look like the following:
15 #
16 # config_opts['system_name'] = "SYSTEM_NAME"
17 # config_opts['restart_cmds'] = [
18 # 'CMD1',
19 # 'CMD2',
20 # ]
21 #
22 # As an example, here is a config file used when monitoring a kvm
23 # instance:
24 #
25 # config_opts['system_name'] = "dhcp-148"
26 # config_opts['restart_cmds'] = [
27 # 'sudo virsh destroy kvm-rawhide-64-1',
28 # 'sudo virsh start kvm-rawhide-64-1',
29 # ]
30 #
31 # Here's a config file used then monitoring a beaker system
32 # (https://beaker-project.org/):
33 #
34 # config_opts['system_name'] = "ibm-z10-38"
35 # config_opts['restart_cmds'] = [
36 # 'bkr system-power --action reboot ibm-z10-38'
37 # ]
38
39 import sys
40 import os
41 import time
42
43 if len(sys.argv) != 2:
44 print >>sys.stderr, "Usage: %s config_file" % sys.argv[0]
45 sys.exit(1)
46 cfg = sys.argv[1]
47
48 # Read in the config file
49 if not os.path.exists(cfg):
50 print >>sys.stderr, ("Could not find required config file: %s" % cfg)
51 sys.exit(1)
52
53 print "Reading config file %s..." % cfg
54 config_opts = dict()
55 execfile(cfg)
56 if not config_opts.has_key('system_name'):
57 print >>sys.stderr, "Missing required config opt 'system_name'"
58 sys.exit(1)
59 if not config_opts.has_key('restart_cmds'):
60 print >>sys.stderr, "Missing required config opt 'restart_cmds'"
61 sys.exit(1)
62
63 errors = 0
64 while 1:
65 rc = os.system("ping -c 1 %s" % config_opts['system_name'])
66 # If ping worked, system is still up and running. Wait a minute
67 # and try again.
68 if os.WEXITSTATUS(rc) == 0:
69 time.sleep(60)
70 errors = 0
71
72 # If the ping failed, increase the error count. If we've got 3
73 # consecutive errors, assume the machine has crashed and restart
74 # it.
75 else:
76 errors += 1
77 if errors < 3:
78 time.sleep(30)
79 else:
80 print >>sys.stderr, "Restarting %s..." % config_opts['system_name']
81 # Run each restart command
82
83 for cmd in config_opts['restart_cmds']:
84 print >>sys.stderr, "Running '%s'..." % cmd
85 os.system(cmd)
86 # Sleep for 5 minutes to give the system a chance to boot
87 print >>sys.stderr, "Sleeping for 5 minutes..."
88 time.sleep(5 * 60)
89 errors = 0
90