| 1 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 2 |
# you may not use this file except in compliance with the License. |
| 3 |
# You may obtain a copy of the License at |
| 4 |
# |
| 5 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 6 |
# |
| 7 |
# Unless required by applicable law or agreed to in writing, software |
| 8 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 9 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 10 |
# See the License for the specific language governing permissions and |
| 11 |
# limitations under the License. |
| 12 |
|
| 13 |
from magnum.conductor import monitors |
| 14 |
from magnum.i18n import _ |
| 15 |
from magnum.objects import fields as m_fields |
| 16 |
from magnum_capi_helm import driver_utils |
| 17 |
from magnum_capi_helm import kubernetes |
| 18 |
|
| 19 |
|
| 20 |
MONITOR_STATE_READY = _("Ready") |
| 21 |
|
| 22 |
|
| 23 |
class CAPIMonitor(monitors.MonitorBase): |
| 24 |
def __init__(self, context, cluster): |
| 25 |
super(CAPIMonitor, self).__init__(context, cluster) |
| 26 |
self.__k8s_client = None |
| 27 |
self.data = {} |
| 28 |
self.data["nodes"] = [] |
| 29 |
self.data["pods"] = [] |
| 30 |
|
| 31 |
@property |
| 32 |
def _k8s_client(self): |
| 33 |
if not self.__k8s_client: |
| 34 |
self.__k8s_client = kubernetes.Client.load() |
| 35 |
return self.__k8s_client |
| 36 |
|
| 37 |
@property |
| 38 |
def metrics_spec(self): |
| 39 |
# TODO(dalees): Not yet implemented for CAPI helm driver |
| 40 |
return super().metrics_spec |
| 41 |
|
| 42 |
def pull_data(self): |
| 43 |
# TODO(dalees): Not yet implemented for CAPI helm driver |
| 44 |
return super().pull_data() |
| 45 |
|
| 46 |
def poll_health_status(self): |
| 47 |
"""Poll health status of API and nodes for given cluster from CAPI |
| 48 |
|
| 49 |
Design Policy: |
| 50 |
1. Magnum should not require direct access to workload clusters. |
| 51 |
This means all health status must come from Cluster API's |
| 52 |
management cluster resources. These already provide health info. |
| 53 |
|
| 54 |
2. Calculating the overall health status. |
| 55 |
We depend on Cluster and OpenstackCluster resources having updated |
| 56 |
'status'. |
| 57 |
From these, we can retrieve overall health and report back. |
| 58 |
|
| 59 |
Some errors surface only in OpenstackCluster state, under |
| 60 |
FailureMessage and FailureReason. Surface this to users, as it can |
| 61 |
be helpful (eg. Neutron 409's when deleting a network that has |
| 62 |
ports) |
| 63 |
|
| 64 |
If we find any Cluster.status.conditions field *not* Status:True, |
| 65 |
then list and set Unhealthy. |
| 66 |
|
| 67 |
If we find OpenstackCluster.status.ready *not* true, report back |
| 68 |
and include FailureReason and FailureMessage if set. |
| 69 |
|
| 70 |
Iterate all nodegroup MachineDeployment and report back on |
| 71 |
status.conditions. |
| 72 |
|
| 73 |
3. The data structure of health_status_reason: |
| 74 |
|
| 75 |
As an attribute of the cluster, the health_status_reason has to |
| 76 |
use the field type from |
| 77 |
oslo.versionedobjects/blob/master/oslo_versionedobjects/fields.py |
| 78 |
|
| 79 |
:return: None. |
| 80 |
|
| 81 |
The class variable data is updated with current status and reason. |
| 82 |
|
| 83 |
""" |
| 84 |
# Start with a good state for everything |
| 85 |
status = m_fields.ClusterHealthStatus.HEALTHY |
| 86 |
reason = {} |
| 87 |
|
| 88 |
for monitor_type, monitor_func in [ |
| 89 |
("cluster", self._poll_cluster), |
| 90 |
("infrastructure", self._poll_infra), |
| 91 |
("controlplane", self._poll_controlplane), |
| 92 |
("nodegroup", self._poll_nodegroups), |
| 93 |
]: |
| 94 |
reason[monitor_type] = monitor_func() |
| 95 |
if reason[monitor_type] != MONITOR_STATE_READY: |
| 96 |
status = m_fields.ClusterHealthStatus.UNHEALTHY |
| 97 |
|
| 98 |
self.data["health_status"] = status |
| 99 |
self.data["health_status_reason"] = reason |
| 100 |
|
| 101 |
def _poll_cluster(self): |
| 102 |
"""Get status from Cluster. |
| 103 |
|
| 104 |
This has most status info available as it bubbles up. |
| 105 |
""" |
| 106 |
namespace = driver_utils.cluster_namespace(self.cluster) |
| 107 |
resource_name = driver_utils.get_k8s_resource_name(self.cluster, None) |
| 108 |
resource_cluster = self._k8s_client.get_capi_cluster( |
| 109 |
resource_name, namespace |
| 110 |
) |
| 111 |
if not resource_cluster: |
| 112 |
return "Cluster resource not found." |
| 113 |
|
| 114 |
cluster_conditions = [ |
| 115 |
c.get("type") |
| 116 |
for c in resource_cluster.get("status", {}).get("conditions", {}) |
| 117 |
if c.get("status") != "True" |
| 118 |
] |
| 119 |
if cluster_conditions: |
| 120 |
return f"Waiting on {cluster_conditions}" |
| 121 |
|
| 122 |
return MONITOR_STATE_READY |
| 123 |
|
| 124 |
def _poll_infra(self): |
| 125 |
"""Analyse OpenStackCluster resource |
| 126 |
|
| 127 |
This represents the CAPO Infrastructure component. |
| 128 |
""" |
| 129 |
namespace = driver_utils.cluster_namespace(self.cluster) |
| 130 |
resource_name = driver_utils.get_k8s_resource_name(self.cluster, None) |
| 131 |
resource_capo = self._k8s_client.get_capi_openstackcluster( |
| 132 |
resource_name, namespace |
| 133 |
) |
| 134 |
if not resource_capo: |
| 135 |
return "Infrastructure resource not found." |
| 136 |
|
| 137 |
capo_status = resource_capo.get("status", {}) |
| 138 |
capo_ready = capo_status.get("ready", False) |
| 139 |
capo_message = capo_status.get("failureMessage", "") |
| 140 |
capo_reason = capo_status.get("failureReason", None) |
| 141 |
|
| 142 |
if capo_reason: |
| 143 |
return f"{capo_reason}: {capo_message}" |
| 144 |
|
| 145 |
if not capo_ready: |
| 146 |
return "Infrastructure not ready." |
| 147 |
|
| 148 |
return MONITOR_STATE_READY |
| 149 |
|
| 150 |
def _poll_controlplane(self): |
| 151 |
"""Analyse KubeAdmControlPlane resource |
| 152 |
|
| 153 |
This CAPI controller manages the control plane machines |
| 154 |
""" |
| 155 |
namespace = driver_utils.cluster_namespace(self.cluster) |
| 156 |
resource_name = driver_utils.get_k8s_resource_name( |
| 157 |
self.cluster, "control-plane" |
| 158 |
) |
| 159 |
resource_kcp = self._k8s_client.get_kubeadm_control_plane( |
| 160 |
resource_name, namespace |
| 161 |
) |
| 162 |
if not resource_kcp: |
| 163 |
return "Control plane resource not found." |
| 164 |
|
| 165 |
conditions = [ |
| 166 |
c.get("type") |
| 167 |
for c in resource_kcp.get("status", {}).get("conditions", {}) |
| 168 |
if c.get("status") != "True" |
| 169 |
] |
| 170 |
if conditions: |
| 171 |
return f"Waiting on {conditions}" |
| 172 |
|
| 173 |
return MONITOR_STATE_READY |
| 174 |
|
| 175 |
def _poll_nodegroups(self): |
| 176 |
"""Analyse MachineDeployment resources |
| 177 |
|
| 178 |
There is one machinedeployment per nodegroup (which holds |
| 179 |
information on machineset and machine resources) |
| 180 |
""" |
| 181 |
namespace = driver_utils.cluster_namespace(self.cluster) |
| 182 |
nodegroup_reasons = [] |
| 183 |
for nodegroup in self.cluster.nodegroups: |
| 184 |
if nodegroup.role == "master": |
| 185 |
continue |
| 186 |
resource_name = driver_utils.get_k8s_resource_name( |
| 187 |
self.cluster, nodegroup.name |
| 188 |
) |
| 189 |
resource_md = self._k8s_client.get_machine_deployment( |
| 190 |
resource_name, namespace |
| 191 |
) |
| 192 |
|
| 193 |
if not resource_md: |
| 194 |
nodegroup_reasons.append( |
| 195 |
f"{nodegroup.name} resource not found." |
| 196 |
) |
| 197 |
continue |
| 198 |
|
| 199 |
conditions = [ |
| 200 |
c.get("type") |
| 201 |
for c in resource_md.get("status", {}).get("conditions", {}) |
| 202 |
if c.get("status") != "True" |
| 203 |
] |
| 204 |
if conditions: |
| 205 |
nodegroup_reasons.append( |
| 206 |
f"{nodegroup.name} waiting on {conditions}" |
| 207 |
) |
| 208 |
if nodegroup_reasons: |
| 209 |
return ",".join(nodegroup_reasons) |
| 210 |
|
| 211 |
return MONITOR_STATE_READY |
| 212 |
|
| 213 |
def _is_cluster_accessible(self): |
| 214 |
# This monitor does not directly access Kubernetes clusters |
| 215 |
# as it obtains all information from CAPI Management cluster. |
| 216 |
# So all clusters are accessible, if CAPI is functional. |
| 217 |
return True |