| 1 |
#!/usr/bin/python |
| 2 |
|
| 3 |
# Copyright (C) 2008, 2014 Red Hat Inc. |
| 4 |
# |
| 5 |
# This file is part of systemtap, and is free software. You can |
| 6 |
# redistribute it and/or modify it under the terms of the GNU General |
| 7 |
# Public License (GPL); either version 2, or (at your option) any |
| 8 |
# later version. |
| 9 |
|
| 10 |
# This script monitors a remote system that is running the kprobes |
| 11 |
# test. If several consecutive 'ping's fail, the system is rebooted. |
| 12 |
# |
| 13 |
# This script takes as an argument a config filename, whose contents |
| 14 |
# should look like the following: |
| 15 |
# |
| 16 |
# config_opts['system_name'] = "SYSTEM_NAME" |
| 17 |
# config_opts['restart_cmds'] = [ |
| 18 |
# 'CMD1', |
| 19 |
# 'CMD2', |
| 20 |
# ] |
| 21 |
# |
| 22 |
# As an example, here is a config file used when monitoring a kvm |
| 23 |
# instance: |
| 24 |
# |
| 25 |
# config_opts['system_name'] = "dhcp-148" |
| 26 |
# config_opts['restart_cmds'] = [ |
| 27 |
# 'sudo virsh destroy kvm-rawhide-64-1', |
| 28 |
# 'sudo virsh start kvm-rawhide-64-1', |
| 29 |
# ] |
| 30 |
# |
| 31 |
# Here's a config file used then monitoring a beaker system |
| 32 |
# (https://beaker-project.org/): |
| 33 |
# |
| 34 |
# config_opts['system_name'] = "ibm-z10-38" |
| 35 |
# config_opts['restart_cmds'] = [ |
| 36 |
# 'bkr system-power --action reboot ibm-z10-38' |
| 37 |
# ] |
| 38 |
|
| 39 |
import sys |
| 40 |
import os |
| 41 |
import time |
| 42 |
|
| 43 |
if len(sys.argv) != 2: |
| 44 |
print >>sys.stderr, "Usage: %s config_file" % sys.argv[0] |
| 45 |
sys.exit(1) |
| 46 |
cfg = sys.argv[1] |
| 47 |
|
| 48 |
# Read in the config file |
| 49 |
if not os.path.exists(cfg): |
| 50 |
print >>sys.stderr, ("Could not find required config file: %s" % cfg) |
| 51 |
sys.exit(1) |
| 52 |
|
| 53 |
print "Reading config file %s..." % cfg |
| 54 |
config_opts = dict() |
| 55 |
execfile(cfg) |
| 56 |
if not config_opts.has_key('system_name'): |
| 57 |
print >>sys.stderr, "Missing required config opt 'system_name'" |
| 58 |
sys.exit(1) |
| 59 |
if not config_opts.has_key('restart_cmds'): |
| 60 |
print >>sys.stderr, "Missing required config opt 'restart_cmds'" |
| 61 |
sys.exit(1) |
| 62 |
|
| 63 |
errors = 0 |
| 64 |
while 1: |
| 65 |
rc = os.system("ping -c 1 %s" % config_opts['system_name']) |
| 66 |
# If ping worked, system is still up and running. Wait a minute |
| 67 |
# and try again. |
| 68 |
if os.WEXITSTATUS(rc) == 0: |
| 69 |
time.sleep(60) |
| 70 |
errors = 0 |
| 71 |
|
| 72 |
# If the ping failed, increase the error count. If we've got 3 |
| 73 |
# consecutive errors, assume the machine has crashed and restart |
| 74 |
# it. |
| 75 |
else: |
| 76 |
errors += 1 |
| 77 |
if errors < 3: |
| 78 |
time.sleep(30) |
| 79 |
else: |
| 80 |
print >>sys.stderr, "Restarting %s..." % config_opts['system_name'] |
| 81 |
# Run each restart command |
| 82 |
|
| 83 |
for cmd in config_opts['restart_cmds']: |
| 84 |
print >>sys.stderr, "Running '%s'..." % cmd |
| 85 |
os.system(cmd) |
| 86 |
# Sleep for 5 minutes to give the system a chance to boot |
| 87 |
print >>sys.stderr, "Sleeping for 5 minutes..." |
| 88 |
time.sleep(5 * 60) |
| 89 |
errors = 0 |
| 90 |
|