Debian Codegraph @

masakarimonitors/hostmonitor/consul_check/manager.py

raw
1 # Copyright(c) 2019 Inspur
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import eventlet
16 import socket
17
18 from collections import deque
19 from oslo_log import log
20 from oslo_utils import timeutils
21
22 from masakarimonitors import conf
23 from masakarimonitors.ha import masakari
24 from masakarimonitors.hostmonitor.consul_check import consul_helper
25 from masakarimonitors.hostmonitor.consul_check import matrix_helper
26 from masakarimonitors.hostmonitor import driver
27 from masakarimonitors.objects import event_constants as ec
28
29 LOG = log.getLogger(__name__)
30 CONF = conf.CONF
31
32
33 class ConsulCheck(driver.DriverBase):
34 """Check host status by consul"""
35
36 def __init__(self):
37 super(ConsulCheck, self).__init__()
38 self.hostname = socket.gethostname()
39 self.monitoring_interval = CONF.host.monitoring_interval
40 self.monitoring_samples = CONF.host.monitoring_samples
41 self.matrix_manager = matrix_helper.MatrixManager(CONF)
42 self.consul_manager = consul_helper.ConsulManager(CONF)
43 self.notifier = masakari.SendNotification()
44 self._matrix = None
45 self._sequence = None
46 self.monitoring_data = {}
47 self.last_host_health = {}
48
49 @property
50 def matrix(self):
51 if not self._matrix:
52 self._matrix = self.matrix_manager.get_matrix()
53 return self._matrix
54
55 @property
56 def sequence(self):
57 if not self._sequence:
58 self._sequence = self.matrix_manager.get_sequence()
59 return self._sequence
60
61 def _formate_health(self, host_health):
62 formate_health = {}
63 for i in range(len(host_health)):
64 layer = "%s-interface" % self.sequence[i]
65 state = host_health[i]
66 formate_health[layer] = state
67
68 return formate_health
69
70 def _event(self, host, host_health):
71 host_status = ec.EventConstants.HOST_STATUS_NORMAL
72 if 'down' not in host_health:
73 event_type = ec.EventConstants.EVENT_STARTED
74 cluster_status = ec.EventConstants.CLUSTER_STATUS_ONLINE
75 else:
76 actions = self.get_action_from_matrix(host_health)
77 if 'recovery' in actions:
78 LOG.info("Host %s needs recovery, health status: %s." %
79 (host, str(self._formate_health(host_health))))
80 event_type = ec.EventConstants.EVENT_STOPPED
81 cluster_status = ec.EventConstants.CLUSTER_STATUS_OFFLINE
82 else:
83 return None
84
85 # TODO(suzhengwei): Add host status detail in the payload to show
86 # host status in all Consul clusters.
87 event = {
88 'notification': {
89 'type': ec.EventConstants.TYPE_COMPUTE_HOST,
90 'hostname': host,
91 'generated_time': timeutils.utcnow(),
92 'payload': {
93 'event': event_type,
94 'cluster_status': cluster_status,
95 'host_status': host_status
96 }
97 }
98 }
99
100 return event
101
102 def update_monitoring_data(self):
103 '''update monitoring data from consul clusters'''
104 LOG.debug("update monitoring data from consul.")
105 # Get current host health in sequence [x, y, z]. The example of
106 # the return value is {'node01':[x, y, z], 'node02':[x, y, z]...}
107 cluster_health = self.consul_manager.get_health(self.sequence)
108 LOG.debug("Current cluster state: %s.", cluster_health)
109 # Reassemble host health history with the latest host health.
110 # Example of 'host_health_history' is [[x, y, x], [x, y, z]...]
111 for host, health in cluster_health.items():
112 host_health_history = self.monitoring_data.get(
113 host, deque([], maxlen=self.monitoring_samples))
114 host_health_history.append(health)
115 self.monitoring_data[host] = host_health_history
116
117 def get_host_health(self, host):
118 health_history = self.monitoring_data.get(host, [])
119 if len(health_history) < self.monitoring_samples:
120 LOG.debug("Not enough monitoring data for host %s", host)
121 return None
122
123 # Caculate host health from host health history.
124 # Only continous 'down' represents the interface 'down',
125 # while continous 'up' represents the interface 'up'.
126 host_sequence_health = []
127 host_health_history = list(zip(*health_history))
128 for i in range(0, len(host_health_history)):
129 if ('up' in host_health_history[i] and
130 'down' in host_health_history[i]):
131 host_sequence_health.append(None)
132 else:
133 host_sequence_health.append(host_health_history[i][0])
134
135 return host_sequence_health
136
137 def _host_health_changed(self, host, health):
138 last_health = self.last_host_health.get(host)
139 if last_health is None:
140 self.last_host_health[host] = health
141 return False
142
143 if health != last_health:
144 self.last_host_health[host] = health
145 return True
146 else:
147 return False
148
149 def get_action_from_matrix(self, host_health):
150 for health_action in self.matrix:
151 matrix_health = health_action["health"]
152 matrix_action = health_action["action"]
153 if host_health == matrix_health:
154 return matrix_action
155
156 return []
157
158 def poll_hosts(self):
159 '''poll and check hosts health'''
160 for host in self.monitoring_data.keys():
161
162 if host == self.hostname:
163 continue
164
165 host_health = self.get_host_health(host)
166 if host_health is None:
167 continue
168
169 if not self._host_health_changed(host, host_health):
170 continue
171
172 # it will send notifition to trigger host failure recovery
173 # according to defined HA strategy
174 event = self._event(host, host_health)
175 if event:
176 self.notifier.send_notification(
177 CONF.host.api_retry_max,
178 CONF.host.api_retry_interval,
179 event)
180
181 def stop(self):
182 self.running = False
183
184 def monitor_hosts(self):
185 self.running = True
186 while self.running:
187 try:
188 self.update_monitoring_data()
189 self.poll_hosts()
190 except Exception as e:
191 LOG.exception("Exception when host-monitor by consul: %s", e)
192
193 eventlet.greenthread.sleep(self.monitoring_interval)