Debian Codegraph @

magnum_capi_helm/driver.py

raw
1 # Licensed under the Apache License, Version 2.0 (the "License"); you may
2 # not use this file except in compliance with the License. You may obtain
3 # a copy of the License at
4 #
5 # http://www.apache.org/licenses/LICENSE-2.0
6 #
7 # Unless required by applicable law or agreed to in writing, software
8 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10 # License for the specific language governing permissions and limitations
11 # under the License.
12 import enum
13 import re
14
15 from magnum.api import utils as api_utils
16 from magnum.common import clients
17 from magnum.common import exception
18 from magnum.common import neutron
19 from magnum.common import short_id
20 from magnum.drivers.common import driver
21 from magnum.objects import fields
22 from oslo_log import log as logging
23 from oslo_utils import strutils
24 from oslo_utils import uuidutils
25
26 from magnum_capi_helm.common import app_creds
27 from magnum_capi_helm.common import ca_certificates
28 from magnum_capi_helm.common import capi_monitor
29 from magnum_capi_helm import conf
30 from magnum_capi_helm import driver_utils
31 from magnum_capi_helm import helm
32 from magnum_capi_helm import kubernetes
33
34 LOG = logging.getLogger(__name__)
35 CONF = conf.CONF
36 NODE_GROUP_ROLE_CONTROLLER = "master"
37
38
39 class NodeGroupState(enum.Enum):
40 NOT_PRESENT = 1
41 PENDING = 2
42 READY = 3
43 FAILED = 4
44
45
46 class Driver(driver.Driver):
47 def __init__(self):
48 self._helm_client = helm.Client()
49 self.__k8s_client = None
50
51 @property
52 def _k8s_client(self):
53 if not self.__k8s_client:
54 self.__k8s_client = kubernetes.Client.load()
55 return self.__k8s_client
56
57 @property
58 def provides(self):
59 return [
60 {
61 "server_type": "vm",
62 # NOTE(johngarbutt) we don't depend on a specific OS,
63 # we depend on kubeadm images with cloud-init
64 "os": "ubuntu",
65 "coe": "kubernetes",
66 },
67 {
68 "server_type": "vm",
69 "os": "flatcar",
70 "coe": "kubernetes",
71 },
72 ]
73
74 def _update_control_plane_nodegroup_status(self, cluster, nodegroup):
75 # The status of the master nodegroup is determined by the Cluster API
76 # control plane object
77 kcp = self._k8s_client.get_kubeadm_control_plane(
78 driver_utils.get_k8s_resource_name(cluster, "control-plane"),
79 driver_utils.cluster_namespace(cluster),
80 )
81
82 ng_state = NodeGroupState.NOT_PRESENT
83 if kcp:
84 ng_state = NodeGroupState.PENDING
85
86 kcp_spec = kcp.get("spec", {}) if kcp else {}
87 kcp_status = kcp.get("status", {}) if kcp else {}
88
89 # The control plane object is what controls the Kubernetes version
90 # If it is known, report it
91 kube_version = kcp_status.get("version", kcp_spec.get("version"))
92 if cluster.coe_version != kube_version:
93 cluster.coe_version = kube_version
94 cluster.save()
95
96 kcp_true_conditions = {
97 cond["type"]
98 for cond in kcp_status.get("conditions", [])
99 if cond["status"] == "True"
100 }
101 kcp_ready = all(
102 cond in kcp_true_conditions
103 for cond in (
104 "MachinesReady",
105 "Ready",
106 "EtcdClusterHealthy",
107 "ControlPlaneComponentsHealthy",
108 )
109 )
110 target_replicas = kcp_spec.get("replicas")
111 current_replicas = kcp_status.get("replicas")
112 updated_replicas = kcp_status.get("updatedReplicas")
113 ready_replicas = kcp_status.get("readyReplicas")
114 if (
115 kcp_ready
116 and target_replicas == current_replicas
117 and current_replicas == updated_replicas
118 and updated_replicas == ready_replicas
119 ):
120 ng_state = NodeGroupState.READY
121
122 # TODO(mkjpryor) Work out a way to determine FAILED state
123 return self._update_nodegroup_status(cluster, nodegroup, ng_state)
124
125 def _update_worker_nodegroup_status(self, cluster, nodegroup):
126 # The status of a worker nodegroup is determined by the corresponding
127 # Cluster API machine deployment
128 md = self._k8s_client.get_machine_deployment(
129 driver_utils.get_k8s_resource_name(cluster, nodegroup.name),
130 driver_utils.cluster_namespace(cluster),
131 )
132
133 ng_state = NodeGroupState.NOT_PRESENT
134 if md:
135 ng_state = NodeGroupState.PENDING
136
137 # When a machine deployment is deleted, it disappears straight
138 # away even when there are still machines belonging to it that
139 # are deleting
140 # In that case, we want to keep the nodegroup as DELETE_IN_PROGRESS
141 # until all the machines for the node group are gone
142 if (
143 not md
144 and nodegroup.status.startswith("DELETE_")
145 and self._nodegroup_machines_exist(cluster, nodegroup)
146 ):
147 LOG.debug(
148 f"Node group {nodegroup.name} "
149 f"for cluster {cluster.uuid} "
150 "machine deployment gone, but machines still found."
151 )
152 ng_state = NodeGroupState.PENDING
153
154 md_status = md.get("status", {}) if md else {}
155 md_phase = md_status.get("phase")
156 if md_phase:
157 if md_phase == "Running":
158 ng_state = NodeGroupState.READY
159 elif md_phase in {"Failed", "Unknown"}:
160 ng_state = NodeGroupState.FAILED
161
162 return self._update_nodegroup_status(cluster, nodegroup, ng_state)
163
164 def _update_nodegroup_status(self, cluster, nodegroup, ng_state):
165 # For delete we are waiting for not present
166 if nodegroup.status.startswith("DELETE_"):
167 if ng_state == NodeGroupState.NOT_PRESENT:
168 if not nodegroup.is_default:
169 # Conductor will delete default nodegroups
170 # when cluster is deleted, but non default
171 # node groups should be deleted here.
172 nodegroup.destroy()
173 LOG.debug(
174 f"Node group deleted: {nodegroup.name} "
175 f"for cluster {cluster.uuid} "
176 f"which is_default: {nodegroup.is_default}"
177 )
178 # signal the node group has been deleted
179 return None
180
181 LOG.debug(
182 f"Node group not yet delete: {nodegroup.name} "
183 f"for cluster {cluster.uuid}"
184 )
185 return nodegroup
186
187 is_update_operation = nodegroup.status.startswith("UPDATE_")
188 is_create_operation = nodegroup.status.startswith("CREATE_")
189 if not is_update_operation and not is_create_operation:
190 LOG.warning(
191 f"Node group: {nodegroup.name} in unexpected "
192 f"state: {nodegroup.status} in cluster {cluster.uuid}"
193 )
194 elif ng_state == NodeGroupState.READY:
195 nodegroup.status = (
196 fields.ClusterStatus.UPDATE_COMPLETE
197 if is_update_operation
198 else fields.ClusterStatus.CREATE_COMPLETE
199 )
200 LOG.debug(
201 f"Node group ready: {nodegroup.name} "
202 f"in cluster {cluster.uuid}"
203 )
204 nodegroup.save()
205
206 elif ng_state == NodeGroupState.FAILED:
207 nodegroup.status = (
208 fields.ClusterStatus.UPDATE_FAILED
209 if is_update_operation
210 else fields.ClusterStatus.CREATE_FAILED
211 )
212 LOG.debug(
213 f"Node group failed: {nodegroup.name} "
214 f"in cluster {cluster.uuid}"
215 )
216 nodegroup.save()
217 elif ng_state == NodeGroupState.NOT_PRESENT:
218 LOG.debug(
219 f"Node group not yet found: {nodegroup.name} "
220 f"state:{nodegroup.status} in cluster {cluster.uuid}"
221 )
222 else:
223 LOG.debug(
224 f"Node group still pending: {nodegroup.name} "
225 f"state:{nodegroup.status} in cluster {cluster.uuid}"
226 )
227
228 return nodegroup
229
230 def _nodegroup_machines_exist(self, cluster, nodegroup):
231 cluster_name = driver_utils.chart_release_name(cluster)
232 nodegroup_name = driver_utils.sanitized_name(nodegroup.name)
233 machines = self._k8s_client.get_all_machines_by_label(
234 {
235 "capi.stackhpc.com/cluster": cluster_name,
236 "capi.stackhpc.com/component": "worker",
237 "capi.stackhpc.com/node-group": nodegroup_name,
238 },
239 driver_utils.cluster_namespace(cluster),
240 )
241 return bool(machines)
242
243 def _update_cluster_api_address(self, cluster, capi_cluster):
244 # As soon as we know the API address, we should set it
245 # This means users can access the API even if the create is
246 # not complete, which could be useful for debugging failures,
247 # e.g. with addons
248 if not capi_cluster:
249 # skip update if cluster not yet created
250 return
251
252 if cluster.status not in [
253 fields.ClusterStatus.CREATE_IN_PROGRESS,
254 fields.ClusterStatus.UPDATE_IN_PROGRESS,
255 ]:
256 # only update api-address when updating or creating
257 return
258
259 api_endpoint = capi_cluster["spec"].get("controlPlaneEndpoint")
260 if api_endpoint:
261 api_address = (
262 f"https://{api_endpoint['host']}:{api_endpoint['port']}"
263 )
264 if cluster.api_address != api_address:
265 cluster.api_address = api_address
266 cluster.save()
267 LOG.debug(f"Found api_address for {cluster.uuid}")
268
269 def _update_status_updating(self, cluster, capi_cluster):
270 # If the cluster is not yet ready then the create/update
271 # is still in progress
272 true_conditions = {
273 cond["type"]
274 for cond in capi_cluster.get("status", {}).get("conditions", [])
275 if cond["status"] == "True"
276 }
277 for cond in ("InfrastructureReady", "ControlPlaneReady", "Ready"):
278 if cond not in true_conditions:
279 return
280
281 is_update_operation = cluster.status.startswith("UPDATE_")
282
283 # Check the status of the addons
284 addons = self._k8s_client.get_addons_by_label(
285 {
286 "addons.stackhpc.com/cluster": driver_utils.chart_release_name(
287 cluster
288 )
289 },
290 driver_utils.cluster_namespace(cluster),
291 )
292 for addon in addons:
293 addon_phase = addon.get("status", {}).get("phase")
294 if addon_phase and addon_phase in {"Failed", "Unknown"}:
295 # If the addon is failed, mark the cluster as failed
296 cluster.status = (
297 fields.ClusterStatus.UPDATE_FAILED
298 if is_update_operation
299 else fields.ClusterStatus.CREATE_FAILED
300 )
301 cluster.save()
302 return
303 elif addon_phase and addon_phase == "Deployed":
304 # If the addon is deployed, move on to the next one
305 continue
306 else:
307 # If there are any addons that are not deployed or failed,
308 # wait for the next invocation to check again
309 LOG.debug(
310 f"addon {addon['metadata']['name']} not yet deployed "
311 f"for {cluster.uuid}"
312 )
313 return
314
315 # If we get this far, the cluster has completed successfully
316 cluster.status = (
317 fields.ClusterStatus.UPDATE_COMPLETE
318 if is_update_operation
319 else fields.ClusterStatus.CREATE_COMPLETE
320 )
321 cluster.save()
322
323 def _update_status_deleting(self, context, cluster):
324 # Once the Cluster API cluster is gone, we need to clean up
325 # the secrets we created
326 self._k8s_client.delete_all_secrets_by_label(
327 "magnum.openstack.org/cluster-uuid",
328 cluster.uuid,
329 driver_utils.cluster_namespace(cluster),
330 )
331
332 # We also need to clean up the appcred that we made
333 app_creds.delete_app_cred(context, cluster)
334
335 cluster.status = fields.ClusterStatus.DELETE_COMPLETE
336 cluster.save()
337
338 def _get_capi_cluster(self, cluster):
339 release_name = driver_utils.chart_release_name(cluster)
340 if release_name:
341 return self._k8s_client.get_capi_cluster(
342 release_name,
343 driver_utils.cluster_namespace(cluster),
344 )
345
346 def _update_all_nodegroups_status(self, cluster):
347 """Returns True if any node group still in progress."""
348 nodegroups = []
349 for nodegroup in cluster.nodegroups:
350 if nodegroup.role == NODE_GROUP_ROLE_CONTROLLER:
351 updated_nodegroup = (
352 self._update_control_plane_nodegroup_status(
353 cluster, nodegroup
354 )
355 )
356 else:
357 updated_nodegroup = self._update_worker_nodegroup_status(
358 cluster, nodegroup
359 )
360 if updated_nodegroup:
361 nodegroups.append(updated_nodegroup)
362
363 # Return True if any are still in progress
364 for nodegroup in nodegroups:
365 if nodegroup.status.endswith("_IN_PROGRESS"):
366 return True
367 return False
368
369 def update_cluster_status(self, context, cluster):
370 # NOTE(mkjpryor)
371 # Because Kubernetes operators are built around reconciliation loops,
372 # Cluster API clusters don't really go into an error state
373 # Hence we only currently handle transitioning from IN_PROGRESS
374 # states to COMPLETE
375
376 # TODO(mkjpryor) Add a timeout for create/update/delete
377
378 capi_cluster = self._get_capi_cluster(cluster)
379
380 if capi_cluster:
381 # Update the cluster API address if it is known
382 # so users can get their coe credentials
383 self._update_cluster_api_address(cluster, capi_cluster)
384
385 # Update the nodegroups first
386 # to ensure API never returns an inconsistent state
387 nodegroups_in_progress = self._update_all_nodegroups_status(
388 cluster
389 )
390
391 if cluster.status in {
392 fields.ClusterStatus.CREATE_IN_PROGRESS,
393 fields.ClusterStatus.UPDATE_IN_PROGRESS,
394 }:
395 LOG.debug("Checking on an update for %s", cluster.uuid)
396 # If the cluster does not exist yet,
397 # create is still in progress
398 if not capi_cluster:
399 LOG.debug(f"capi_cluster not yet created for {cluster.uuid}")
400 return
401 if nodegroups_in_progress:
402 LOG.debug(f"Node groups are not all ready for {cluster.uuid}")
403 return
404 self._update_status_updating(cluster, capi_cluster)
405
406 elif cluster.status == fields.ClusterStatus.DELETE_IN_PROGRESS:
407 LOG.debug("Checking on a delete for %s", cluster.uuid)
408 # If the Cluster API cluster still exists,
409 # the delete is still in progress
410 if capi_cluster:
411 LOG.debug(f"capi_cluster still found for {cluster.uuid}")
412 return
413 self._update_status_deleting(context, cluster)
414
415 def get_monitor(self, context, cluster):
416 return capi_monitor.CAPIMonitor(context, cluster)
417
418 def _k8s_resource_labels(self, cluster):
419 # TODO(johngarbutt) need to check these are safe labels
420 name = driver_utils.chart_release_name(cluster)
421 return {
422 "magnum.openstack.org/project-id": cluster.project_id[:63],
423 "magnum.openstack.org/user-id": cluster.user_id[:63],
424 "magnum.openstack.org/cluster-uuid": cluster.uuid[:63],
425 "cluster.x-k8s.io/cluster-name": name,
426 }
427
428 def _create_appcred_secret(self, context, cluster):
429 string_data = app_creds.get_app_cred_string_data(context, cluster)
430 name = self._get_app_cred_name(cluster)
431 self._k8s_client.apply_secret(
432 name,
433 {
434 "metadata": {"labels": self._k8s_resource_labels(cluster)},
435 "stringData": string_data,
436 },
437 driver_utils.cluster_namespace(cluster),
438 )
439
440 def _ensure_certificate_secrets(self, context, cluster):
441 # Magnum creates CA certs for each of the Kubernetes components that
442 # must be trusted by the cluster
443 # In particular, this is required for "openstack coe cluster config"
444 # to work, as that doesn't communicate with the driver and instead
445 # relies on the correct CA being trusted by the cluster
446
447 # Cluster API looks for specific named secrets for each of the CAs,
448 # and generates them if they don't exist, so we create them here
449 # with the correct certificates in
450 for (
451 name,
452 data,
453 ) in ca_certificates.get_certificate_string_data(
454 context, cluster
455 ).items():
456 self._k8s_client.apply_secret(
457 driver_utils.get_k8s_resource_name(cluster, name),
458 {
459 "metadata": {"labels": self._k8s_resource_labels(cluster)},
460 "type": "cluster.x-k8s.io/secret",
461 "stringData": data,
462 },
463 driver_utils.cluster_namespace(cluster),
464 )
465
466 def _label(self, cluster, key, default):
467 all_labels = helm.mergeconcat(
468 cluster.cluster_template.labels, cluster.labels
469 )
470 if not all_labels:
471 return default
472 raw = all_labels.get(key, default)
473 # NOTE(johngarbutt): filtering untrusted user input
474 return re.sub(r"[^a-zA-Z0-9\.\-\/ _]+", "", raw)
475
476 def _get_label_bool(self, cluster, label, default):
477 cluster_label = self._label(cluster, label, "")
478 return strutils.bool_from_string(cluster_label, default=default)
479
480 def _get_label_int(self, cluster, label, default):
481 cluster_label = self._label(cluster, label, "")
482 if not cluster_label:
483 return default
484 try:
485 return int(cluster_label)
486 except ValueError:
487 return default
488
489 def _get_chart_version(self, cluster):
490 version = cluster.cluster_template.labels.get(
491 "capi_helm_chart_version",
492 CONF.capi_helm.default_helm_chart_version,
493 )
494 # NOTE(johngarbutt): filtering untrusted user input
495 return re.sub(r"[^a-z0-9\.\-\+]+", "", version)
496
497 def _get_kube_version(self, image):
498 # The image should have a property containing the Kubernetes version
499 kube_version = image.get("kube_version")
500 if not kube_version:
501 raise exception.MagnumException(
502 message=f"Image {image.id} does not "
503 "have a kube_version property."
504 )
505 raw = kube_version.lstrip("v")
506 # TODO(johngarbutt) more validation required?
507 return re.sub(r"[^0-9\.]+", "", raw)
508
509 def _get_os_distro(self, image):
510 os_distro = image.get("os_distro")
511 if not os_distro:
512 raise exception.MagnumException(
513 message=f"Image {image.id} does not "
514 "have an os_distro property."
515 )
516 return re.sub(r"[^a-zA-Z0-9\.\-\/ ]+", "", os_distro)
517
518 def _get_image_details(self, context, image_identifier):
519 osc = clients.OpenStackClients(context)
520 image = api_utils.get_openstack_resource(
521 osc.glance().images, image_identifier, "images"
522 )
523 return (
524 image.id,
525 self._get_kube_version(image),
526 self._get_os_distro(image),
527 )
528
529 def _get_app_cred_name(self, cluster):
530 return driver_utils.get_k8s_resource_name(cluster, "cloud-credentials")
531
532 def _get_etcd_config(self, cluster):
533 # Support new-style and legacy labels for volume size and type, with
534 # new-style labels taking precedence
535 etcd_size = self._get_label_int(
536 cluster,
537 "etcd_blockdevice_size",
538 self._get_label_int(cluster, "etcd_volume_size", 0),
539 )
540 if etcd_size > 0:
541 etcd_block_device = {"size": etcd_size}
542 # The block device type can be either local or volume
543 etcd_bd_type = self._label(
544 cluster, "etcd_blockdevice_type", "volume"
545 )
546 if etcd_bd_type == "local":
547 etcd_block_device["type"] = "Local"
548 else:
549 etcd_block_device["type"] = "Volume"
550
551 etcd_volume_type = self._label(
552 cluster,
553 "etcd_blockdevice_volume_type",
554 self._label(cluster, "etcd_volume_type", ""),
555 )
556 if etcd_volume_type:
557 etcd_block_device["volumeType"] = etcd_volume_type
558
559 etcd_volume_az = self._label(
560 cluster, "etcd_blockdevice_volume_az", ""
561 )
562 if etcd_volume_az:
563 etcd_block_device["availabilityZone"] = etcd_volume_az
564 return {"blockDevice": etcd_block_device}
565 else:
566 return {}
567
568 def _get_dns_nameservers(self, cluster):
569 dns_nameserver = cluster.cluster_template.dns_nameserver
570 if dns_nameserver:
571 return strutils.split_by_commas(dns_nameserver)
572 else:
573 return None
574
575 def _get_monitoring_enabled(self, cluster):
576 # NOTE(mkjpryor) default off, like heat driver,
577 # as requires cinder and takes a while
578 return self._get_label_bool(cluster, "monitoring_enabled", False)
579
580 def _get_kube_dash_enabled(self, cluster):
581 # NOTE(mkjpryor) default on, like the heat driver
582 return self._get_label_bool(cluster, "kube_dashboard_enabled", True)
583
584 def _get_autoheal_enabled(self, cluster):
585 return self._get_label_bool(cluster, "auto_healing_enabled", True)
586
587 def _get_autoscale_enabled(self, cluster):
588 return self._get_label_bool(cluster, "auto_scaling_enabled", False)
589
590 def _get_autoscale_values(self, cluster, nodegroup):
591 auto_scale = self._get_autoscale_enabled(cluster)
592 min_nodes, max_nodes = self._validate_allowed_node_counts(
593 cluster, nodegroup
594 )
595 auto_scale_args = {}
596 if auto_scale and min_nodes != max_nodes:
597 auto_scale_args["autoscale"] = "true"
598 auto_scale_args["machineCountMin"] = min_nodes
599 auto_scale_args["machineCountMax"] = max_nodes
600 return auto_scale_args
601
602 def _get_k8s_keystone_auth_enabled(self, cluster):
603 return self._get_label_bool(cluster, "keystone_auth_enabled", False)
604
605 def _get_fixed_network_id(self, context, cluster):
606 network = cluster.fixed_network
607 if not network:
608 return
609 if network and uuidutils.is_uuid_like(network):
610 return network
611 else:
612 return neutron.get_network(
613 context, network, source="name", target="id", external=False
614 )
615
616 def _validate_allowed_flavor(self, context, requested_flavor):
617 # Compare requested flavor with allowed for Kubernetes node
618 flavors = (
619 clients.OpenStackClients(context)
620 .nova()
621 .flavors.list(min_ram=CONF.capi_helm.minimum_flavor_ram)
622 )
623 for flavor in flavors:
624 vcpus = flavor.vcpus
625 LOG.debug(
626 f"Checking if {requested_flavor} matches "
627 f"{flavor.id} or {flavor.name}"
628 )
629 if requested_flavor in [flavor.id, flavor.name]:
630 if vcpus < CONF.capi_helm.minimum_flavor_vcpus:
631 raise exception.MagnumException(
632 message=f"Flavor {requested_flavor} does not "
633 f"have enough CPU to run Kubernetes. "
634 f"Minimum {CONF.capi_helm.minimum_flavor_vcpus} "
635 "vcpus required."
636 )
637 return
638 raise exception.MagnumException(
639 message=f"Flavor {requested_flavor} does not "
640 f"have enough RAM to run Kubernetes. "
641 f"Minimum {CONF.capi_helm.minimum_flavor_ram} MB required."
642 )
643
644 def _is_default_worker_nodegroup(self, cluster, nodegroup):
645 return cluster.default_ng_worker.id == nodegroup.id
646
647 def _get_node_counts(self, cluster, nodegroup):
648
649 # NOTE(scott): In CAPI MachineDeployment resources created by the
650 # capi-helm-charts, the `replicas` field is omitted when autoscaling
651 # is enabled (since we're relinquishing control over node count to
652 # the autoscaler) so if a user creates a nodegroup where only
653 # node_count is provided and min/max are not, we need to be careful
654 # about setting the default min/max values to equal node_count.
655 min_nodes = nodegroup.node_count
656 max_nodes = nodegroup.node_count
657
658 if nodegroup.min_node_count is not None:
659 min_nodes = nodegroup.min_node_count
660 if nodegroup.max_node_count is not None:
661 max_nodes = nodegroup.max_node_count
662
663 # If min/max node counts are not defined on the default
664 # worker group then fall back to equivalent cluster labels
665 if self._is_default_worker_nodegroup(cluster, nodegroup):
666 # NOTE(scott): Magnum seems to set min_node_count = 1
667 # on the default group so treat this as if it were None
668 if nodegroup.min_node_count == 1:
669 min_nodes = nodegroup.node_count
670
671 # We still want to be able to override the default node
672 # group values with labels for consistent behaviour with
673 # Magnum Heat driver.
674 min_nodes = self._get_label_int(
675 cluster, "min_node_count", min_nodes
676 )
677 max_nodes = self._get_label_int(
678 cluster, "max_node_count", max_nodes
679 )
680
681 return min_nodes, max_nodes
682
683 def _validate_allowed_node_counts(self, cluster, nodegroup):
684 min_nodes, max_nodes = self._get_node_counts(cluster, nodegroup)
685
686 LOG.debug(
687 f"Checking if node group {nodegroup.name} has valid "
688 f"node count parameters (count, min, max) = "
689 f"{(nodegroup.node_count, min_nodes, max_nodes)}"
690 )
691
692 if min_nodes is not None:
693 # ClusterAPI Provider OpenStack (CAPO)
694 # doesn't support scale to zero yet.
695 if min_nodes < 1:
696 raise exception.NodeGroupInvalidInput(
697 message="Min node count must be greater than "
698 "or equal to 1 for all node groups."
699 )
700 if min_nodes > nodegroup.node_count:
701 raise exception.NodeGroupInvalidInput(
702 message="Min node count must be less than "
703 "or equal to current node count"
704 )
705 if max_nodes is not None and max_nodes < min_nodes:
706 raise exception.NodeGroupInvalidInput(
707 message="Max node count must be greater than "
708 "or equal to min node count"
709 )
710
711 return min_nodes, max_nodes
712
713 def _get_csi_cinder_availability_zone(self, cluster):
714 return self._label(
715 cluster,
716 "csi_cinder_availability_zone",
717 CONF.capi_helm.csi_cinder_availability_zone,
718 )
719
720 def _get_csi_cinder_reclaim_policy(self, cluster):
721 return self._label(
722 cluster,
723 "csi_cinder_reclaim_policy",
724 CONF.capi_helm.csi_cinder_reclaim_policy,
725 )
726
727 def _get_csi_cinder_fstype(self, cluster):
728 return self._label(
729 cluster,
730 "csi_cinder_fstype",
731 CONF.capi_helm.csi_cinder_fstype,
732 )
733
734 def _get_csi_cinder_allow_volume_expansion(self, cluster):
735 return self._get_label_bool(
736 cluster,
737 "csi_cinder_allow_volume_expansion",
738 CONF.capi_helm.csi_cinder_allow_volume_expansion,
739 )
740
741 def _get_octavia_provider(self, cluster):
742 return self._label(cluster, "octavia_provider", "amphora")
743
744 def _get_octavia_lb_algorithm(self, cluster):
745 provider = self._get_octavia_provider(cluster)
746 return self._label(
747 cluster,
748 "octavia_lb_algorithm",
749 "SOURCE_IP_PORT" if provider.lower() == "ovn" else "ROUND_ROBIN",
750 )
751
752 def _get_allowed_cidrs(self, cluster):
753 cidr_list = cluster.labels.get("api_master_lb_allowed_cidrs", "")
754 LOG.debug(f"CIDR list {cidr_list}")
755 if isinstance(cidr_list, str) and cidr_list != "":
756 return cidr_list.split(",")
757 return False
758
759 def _storageclass_definitions(self, context, cluster):
760 """Query cinder API to retrieve list of available volume types.
761
762 @return dict(dict,list(dict)) containing storage classes
763 """
764 LOG.debug("Retrieve volume types from cinder for StorageClasses.")
765 client = clients.OpenStackClients(context)
766 availability_zone = self._get_csi_cinder_availability_zone(cluster)
767 c_client = client.cinder()
768 volume_types = [i.name for i in c_client.volume_types.list()]
769 # Use the default volume type if defined. Otherwise use the first
770 # type returned by cinder.
771 default_volume_type = CONF.capi_helm.csi_cinder_default_volume_type
772 LOG.debug(
773 f"Default volume type: {default_volume_type}"
774 f" Volume types: {volume_types}"
775 )
776 if not default_volume_type:
777 default_volume_type = volume_types[0]
778 LOG.warning(
779 f"Default volume type not defined."
780 f" Using {default_volume_type}."
781 )
782 elif default_volume_type not in volume_types:
783 # If default does not exist throw an error.
784 raise exception.MagnumException(
785 message=f"{default_volume_type} is not a"
786 " valid Cinder volume type."
787 )
788 default_storage_class = {}
789 additional_storage_classes = []
790 allow_expansion = self._get_csi_cinder_allow_volume_expansion(cluster)
791 reclaim_policy = self._get_csi_cinder_reclaim_policy(cluster)
792 allowed_topologies = CONF.capi_helm.csi_cinder_allowed_topologies
793 fstype = self._get_csi_cinder_fstype(cluster)
794
795 for volume_type in volume_types:
796 storage_class = {
797 "name": driver_utils.sanitized_name(volume_type),
798 "reclaimPolicy": reclaim_policy,
799 "allowVolumeExpansion": allow_expansion,
800 "availabilityZone": availability_zone,
801 "volumeType": volume_type,
802 "allowedTopologies": allowed_topologies,
803 "fstype": fstype,
804 "enabled": True,
805 }
806 if volume_type == default_volume_type:
807 default_storage_class = storage_class
808 else:
809 additional_storage_classes.append(storage_class)
810 return dict(
811 defaultStorageClass=default_storage_class,
812 additionalStorageClasses=additional_storage_classes,
813 )
814
815 def _process_node_groups(self, cluster):
816 nodegroups = cluster.nodegroups
817 nodegroup_set = []
818 for ng in nodegroups:
819 if ng.role != NODE_GROUP_ROLE_CONTROLLER:
820 nodegroup_item = dict(
821 name=driver_utils.sanitized_name(ng.name),
822 machineFlavor=ng.flavor_id,
823 machineCount=ng.node_count,
824 )
825 if self._get_autoscale_enabled(cluster):
826 values = self._get_autoscale_values(cluster, ng)
827 nodegroup_item = helm.mergeconcat(nodegroup_item, values)
828 nodegroup_set.append(nodegroup_item)
829 return nodegroup_set
830
831 def _update_helm_release(self, context, cluster, nodegroups=None):
832 if nodegroups is None:
833 nodegroups = cluster.nodegroups
834
835 image_id, kube_version, os_distro = self._get_image_details(
836 context, cluster.cluster_template.image_id
837 )
838
839 network_id = self._get_fixed_network_id(context, cluster)
840 subnet_id = neutron.get_fixed_subnet_id(context, cluster.fixed_subnet)
841
842 values = {
843 "kubernetesVersion": kube_version,
844 "machineImageId": image_id,
845 "machineSSHKeyName": cluster.keypair or None,
846 "cloudCredentialsSecretName": self._get_app_cred_name(cluster),
847 "etcd": self._get_etcd_config(cluster),
848 "apiServer": {
849 "enableLoadBalancer": True,
850 "loadBalancerProvider": self._get_octavia_provider(cluster),
851 },
852 "clusterNetworking": {
853 "dnsNameservers": self._get_dns_nameservers(cluster),
854 "externalNetworkId": neutron.get_external_network_id(
855 context, cluster.cluster_template.external_network_id
856 ),
857 "internalNetwork": {
858 "networkFilter": (
859 {"id": network_id} if network_id else None
860 ),
861 "subnetFilter": ({"id": subnet_id} if subnet_id else None),
862 # This is only used if a fixed network is not specified
863 "nodeCidr": self._label(
864 cluster, "fixed_subnet_cidr", "10.0.0.0/24"
865 ),
866 },
867 },
868 "osDistro": os_distro,
869 "controlPlane": {
870 "machineFlavor": cluster.master_flavor_id,
871 "machineCount": cluster.master_count,
872 "healthCheck": {
873 "enabled": self._get_autoheal_enabled(cluster),
874 },
875 },
876 "nodeGroupDefaults": {
877 "healthCheck": {
878 "enabled": self._get_autoheal_enabled(cluster),
879 },
880 },
881 "nodeGroups": self._process_node_groups(cluster),
882 "addons": {
883 "openstack": {
884 "csiCinder": self._storageclass_definitions(
885 context, cluster
886 ),
887 "cloudConfig": {
888 "LoadBalancer": {
889 "lb-provider": self._get_octavia_provider(cluster),
890 "lb-method": self._get_octavia_lb_algorithm(
891 cluster
892 ),
893 "create-monitor": self._get_label_bool(
894 cluster, "octavia_lb_healthcheck", True
895 ),
896 }
897 },
898 },
899 "monitoring": {
900 "enabled": self._get_monitoring_enabled(cluster)
901 },
902 "kubernetesDashboard": {
903 "enabled": self._get_kube_dash_enabled(cluster)
904 },
905 # TODO(mkjpryor): can't enable ingress until code exists to
906 # remove the load balancer
907 "ingress": {"enabled": False},
908 },
909 }
910
911 # Add boot disk details, if defined in config file.
912 # Helm chart defaults to ephemeral disks, if unset.
913 boot_volume_type = self._label(
914 cluster, "boot_volume_type", CONF.cinder.default_boot_volume_type
915 )
916 if boot_volume_type:
917 disk_type_details = {
918 "controlPlane": {
919 "machineRootVolume": {
920 "volumeType": boot_volume_type,
921 }
922 },
923 "nodeGroupDefaults": {
924 "machineRootVolume": {
925 "volumeType": boot_volume_type,
926 }
927 },
928 }
929 values = helm.mergeconcat(values, disk_type_details)
930
931 boot_volume_size_gb = self._get_label_int(
932 cluster, "boot_volume_size", CONF.cinder.default_boot_volume_size
933 )
934 if boot_volume_size_gb:
935 disk_size_details = {
936 "controlPlane": {
937 "machineRootVolume": {
938 "diskSize": boot_volume_size_gb,
939 }
940 },
941 "nodeGroupDefaults": {
942 "machineRootVolume": {
943 "diskSize": boot_volume_size_gb,
944 }
945 },
946 }
947 values = helm.mergeconcat(values, disk_size_details)
948
949 # Sometimes you need to add an extra network
950 # for things like Cinder CSI CephFS Native
951 extra_network_name = self._label(cluster, "extra_network_name", "")
952 if extra_network_name:
953 network_details = {
954 "nodeGroupDefaults": {
955 "machineNetworking": {
956 "ports": [
957 {},
958 {
959 "network": {
960 "name": extra_network_name,
961 },
962 "securityGroups": [],
963 },
964 ],
965 },
966 },
967 }
968 values = helm.mergeconcat(values, network_details)
969
970 if self._get_k8s_keystone_auth_enabled(cluster):
971 k8s_keystone_auth_config = {
972 "authWebhook": "k8s-keystone-auth",
973 "addons": {
974 "openstack": {
975 "k8sKeystoneAuth": { # addon subchart configuration
976 "enabled": True,
977 "values": {
978 "openstackAuthUrl": context.auth_url,
979 "projectId": context.project_id,
980 },
981 }
982 }
983 },
984 }
985 values = helm.mergeconcat(values, k8s_keystone_auth_config)
986 LOG.debug(
987 "Enable K8s keystone auth webhook for"
988 f" project: {context.project_id} auth url: {context.auth_url}"
989 )
990
991 api_lb_allowed_cidrs = self._get_allowed_cidrs(cluster)
992 if isinstance(api_lb_allowed_cidrs, list):
993 allowed_cidrs_config = {
994 "apiServer": {"allowedCidrs": api_lb_allowed_cidrs}
995 }
996 values = helm.mergeconcat(values, allowed_cidrs_config)
997
998 self._helm_client.install_or_upgrade(
999 driver_utils.chart_release_name(cluster),
1000 CONF.capi_helm.helm_chart_name,
1001 values,
1002 repo=CONF.capi_helm.helm_chart_repo,
1003 version=self._get_chart_version(cluster),
1004 namespace=driver_utils.cluster_namespace(cluster),
1005 )
1006
1007 def _generate_release_name(self, cluster):
1008 if cluster.stack_id:
1009 return
1010
1011 # Make sure no duplicate names
1012 # by generating 12 character random id
1013 random_bit = short_id.generate_id()
1014 base_name = driver_utils.sanitized_name(cluster.name)
1015 # valid release names are 53 chars long
1016 # and stack_id is 12 characters
1017 # but we also use this to derive hostnames
1018 trimmed_name = base_name[:30]
1019 # Save the full name, so users can rename in the API
1020 cluster.stack_id = f"{trimmed_name}-{random_bit}".lower()
1021 # be sure to save this before we use it
1022 cluster.save()
1023
1024 def create_cluster(self, context, cluster, cluster_create_timeout):
1025 LOG.info("Starting to create cluster %s", cluster.uuid)
1026
1027 self._validate_allowed_flavor(context, cluster.master_flavor_id)
1028 nodegroups = cluster.nodegroups
1029 for ng in nodegroups:
1030 self._validate_allowed_flavor(context, ng.flavor_id)
1031 # we generate this name (on the initial create call only)
1032 # so we hit no issues with duplicate cluster names
1033 # and it makes renaming clusters in the API possible
1034 self._generate_release_name(cluster)
1035
1036 # NOTE(johngarbutt) all node groups should already
1037 # be in the CREATE_IN_PROGRESS state
1038 self._k8s_client.ensure_namespace(
1039 driver_utils.cluster_namespace(cluster)
1040 )
1041 self._create_appcred_secret(context, cluster)
1042 self._ensure_certificate_secrets(context, cluster)
1043
1044 self._update_helm_release(context, cluster)
1045
1046 def update_cluster(
1047 self, context, cluster, scale_manager=None, rollback=False
1048 ):
1049 # Cluster API refuses to update things like cluster networking,
1050 # so it is safest not to implement this for now
1051 # TODO(mkjpryor) Check what bits of update we can support
1052 raise NotImplementedError(
1053 "Updating a cluster in this way is not currently supported"
1054 )
1055
1056 def delete_cluster(self, context, cluster):
1057 LOG.info("Starting to delete cluster %s", cluster.uuid)
1058
1059 # Copy the helm driver by marking all node groups
1060 # as delete in progress here, as note done by conductor
1061 # We do this before calling uninstall_release because
1062 # update_cluster_status can get called before we return
1063 for ng in cluster.nodegroups:
1064 ng.status = fields.ClusterStatus.DELETE_IN_PROGRESS
1065 ng.save()
1066
1067 release_name = driver_utils.chart_release_name(cluster)
1068 # Only attempt deletion of CAPI resources if they were created in
1069 # the first place e.g. if trust creation fails during cluster create
1070 # then no CAPI resources will have been created.
1071 if release_name:
1072 # Begin the deletion of the cluster resources by uninstalling the
1073 # Helm release.
1074 # Note that this just marks the resources for deletion,
1075 # it does not wait for the resources to be deleted.
1076 self._helm_client.uninstall_release(
1077 release_name,
1078 namespace=driver_utils.cluster_namespace(cluster),
1079 )
1080
1081 def resize_cluster(
1082 self,
1083 context,
1084 cluster,
1085 resize_manager,
1086 node_count,
1087 nodes_to_remove,
1088 nodegroup=None,
1089 ):
1090 if nodes_to_remove:
1091 LOG.warning("Removing specific nodes is not currently supported")
1092 self._update_helm_release(context, cluster)
1093
1094 def upgrade_cluster(
1095 self,
1096 context,
1097 cluster,
1098 cluster_template,
1099 max_batch_size,
1100 nodegroup,
1101 scale_manager=None,
1102 rollback=False,
1103 ):
1104 # TODO(mkjpryor) check that the upgrade is viable
1105 # e.g. not a downgrade, not an upgrade by more than one minor version
1106
1107 # Updating the template will likely apply for all nodegroups
1108 # So mark them all as having an update in progress
1109 for nodegroup in cluster.nodegroups:
1110 nodegroup.status = fields.ClusterStatus.UPDATE_IN_PROGRESS
1111 self._validate_allowed_flavor(context, nodegroup.flavor_id)
1112 nodegroup.save()
1113
1114 # Move the cluster to the new template
1115 cluster.cluster_template_id = cluster_template.uuid
1116 cluster.status = fields.ClusterStatus.UPDATE_IN_PROGRESS
1117 cluster.save()
1118 cluster.refresh()
1119
1120 self._update_helm_release(context, cluster)
1121
1122 def create_nodegroup(self, context, cluster, nodegroup):
1123 nodegroup.status = fields.ClusterStatus.CREATE_IN_PROGRESS
1124 self._validate_allowed_flavor(context, nodegroup.flavor_id)
1125 if self._get_autoscale_enabled(cluster):
1126 self._validate_allowed_node_counts(cluster, nodegroup)
1127 nodegroup.save()
1128
1129 self._update_helm_release(context, cluster)
1130
1131 def update_nodegroup(self, context, cluster, nodegroup):
1132 nodegroup.status = fields.ClusterStatus.UPDATE_IN_PROGRESS
1133 self._validate_allowed_flavor(context, nodegroup.flavor_id)
1134 if self._get_autoscale_enabled(cluster):
1135 self._validate_allowed_node_counts(cluster, nodegroup)
1136 nodegroup.save()
1137
1138 self._update_helm_release(context, cluster)
1139
1140 def delete_nodegroup(self, context, cluster, nodegroup):
1141 nodegroup.status = fields.ClusterStatus.DELETE_IN_PROGRESS
1142 nodegroup.save()
1143
1144 # Remove the nodegroup being deleted from the nodegroups
1145 # for the Helm release
1146 self._update_helm_release(
1147 context,
1148 cluster,
1149 [ng for ng in cluster.nodegroups if ng.name != nodegroup.name],
1150 )
1151
1152 def create_federation(self, context, federation):
1153 raise NotImplementedError("Will not implement 'create_federation'")
1154
1155 def update_federation(self, context, federation):
1156 raise NotImplementedError("Will not implement 'update_federation'")
1157
1158 def delete_federation(self, context, federation):
1159 raise NotImplementedError("Will not implement 'delete_federation'")