| 1 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 2 |
# not use this file except in compliance with the License. You may obtain |
| 3 |
# a copy of the License at |
| 4 |
# |
| 5 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 6 |
# |
| 7 |
# Unless required by applicable law or agreed to in writing, software |
| 8 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 9 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 10 |
# License for the specific language governing permissions and limitations |
| 11 |
# under the License. |
| 12 |
|
| 13 |
import fixtures |
| 14 |
|
| 15 |
import kubernetes |
| 16 |
from kubernetes.client.rest import ApiException |
| 17 |
from oslo_log import log as logging |
| 18 |
from oslo_serialization import base64 |
| 19 |
from oslo_utils import uuidutils |
| 20 |
from tempest.lib.common.utils import data_utils |
| 21 |
from tempest.lib import decorators |
| 22 |
from tempest.lib import exceptions |
| 23 |
import testtools |
| 24 |
from yaml import safe_load |
| 25 |
|
| 26 |
from magnum_tempest_plugin.common import config |
| 27 |
from magnum_tempest_plugin.common import datagen |
| 28 |
from magnum_tempest_plugin.common import utils |
| 29 |
from magnum_tempest_plugin.tests.api import base |
| 30 |
|
| 31 |
|
| 32 |
HEADERS = {'OpenStack-API-Version': 'container-infra latest', |
| 33 |
'Accept': 'application/json', |
| 34 |
'Content-Type': 'application/json'} |
| 35 |
|
| 36 |
# Please overwrite this ID map to specific IDs for tests |
| 37 |
IDEMPOTENT_IDS = { |
| 38 |
"test_delete_cluster_for_nonexisting_cluster": ( |
| 39 |
"208fabdb-f7b8-4bdd-8762-fdf88be0fbca"), |
| 40 |
"test_update_cluster_for_nonexisting_cluster": ( |
| 41 |
"71c75c03-090c-4f33-9aad-f2919a3b63d2"), |
| 42 |
"test_create_cluster_with_nonexisting_flavor": ( |
| 43 |
"5e4e68d7-6b0a-4548-a462-2f5c83004509"), |
| 44 |
"test_create_cluster_with_zero_masters": ( |
| 45 |
"395288a7-dc5d-474b-ba2b-09158553d549"), |
| 46 |
"test_create_cluster_with_zero_nodes": ( |
| 47 |
"6201ac61-fd28-43a9-824a-2d452b2fe5c5"), |
| 48 |
"test_create_cluster_for_nonexisting_cluster_template": ( |
| 49 |
"1c81a8b2-de86-4494-a8a7-b194f4c1a1f9"), |
| 50 |
"test_create_list_sign_delete_clusters": ( |
| 51 |
"82bdca24-d732-41d1-bfc0-983ec8229ca0"), |
| 52 |
} |
| 53 |
|
| 54 |
|
| 55 |
class ClusterTestTemplate(base.BaseTempestTest): |
| 56 |
|
| 57 |
"""Tests template for cluster CRUD.""" |
| 58 |
|
| 59 |
LOG = logging.getLogger(__name__) |
| 60 |
delete_template = False |
| 61 |
|
| 62 |
def __init__(self, *args, **kwargs): |
| 63 |
super().__init__(*args, **kwargs) |
| 64 |
self.clusters = [] |
| 65 |
|
| 66 |
@classmethod |
| 67 |
def _setup_cluster_template(cls): |
| 68 |
if config.Config.cluster_template_id: |
| 69 |
_, cls.cluster_template = \ |
| 70 |
cls.ct_reader_client.get_cluster_template( |
| 71 |
config.Config.cluster_template_id) |
| 72 |
else: |
| 73 |
model = datagen.valid_cluster_template() |
| 74 |
_, cls.cluster_template = cls._create_cluster_template(model) |
| 75 |
cls.delete_template = True |
| 76 |
|
| 77 |
def setUp(self): |
| 78 |
super().setUp() |
| 79 |
|
| 80 |
# NOTE (dimtruck) by default tempest sets timeout to 20 mins. |
| 81 |
# We need more time. |
| 82 |
test_timeout = 3600 |
| 83 |
self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) |
| 84 |
|
| 85 |
def tearDown(self): |
| 86 |
try: |
| 87 |
cluster_list = self.clusters[:] |
| 88 |
for cluster_id in cluster_list: |
| 89 |
self._delete_cluster(cluster_id) |
| 90 |
self.clusters.remove(cluster_id) |
| 91 |
finally: |
| 92 |
super().tearDown() |
| 93 |
|
| 94 |
@classmethod |
| 95 |
def _create_cluster_template(cls, cm_model): |
| 96 |
cls.LOG.debug('We will create a clustertemplate for %s', cm_model) |
| 97 |
resp, model = cls.ct_member_client.post_cluster_template( |
| 98 |
cm_model) |
| 99 |
return resp, model |
| 100 |
|
| 101 |
@classmethod |
| 102 |
def _delete_cluster_template(cls, cm_id): |
| 103 |
cls.LOG.debug('We will delete a clustertemplate for %s', cm_id) |
| 104 |
resp, model = cls.ct_member_client.delete_cluster_template( |
| 105 |
cm_id) |
| 106 |
return resp, model |
| 107 |
|
| 108 |
def _create_cluster(self, cluster_model): |
| 109 |
self.LOG.debug('We will create cluster for %s', cluster_model) |
| 110 |
resp, model = self.cluster_member_client.post_cluster(cluster_model) |
| 111 |
self.LOG.debug('Response: %s', resp) |
| 112 |
self.assertEqual(202, resp.status) |
| 113 |
self.assertIsNotNone(model.uuid) |
| 114 |
self.assertTrue(uuidutils.is_uuid_like(model.uuid)) |
| 115 |
self.clusters.append(model.uuid) |
| 116 |
self.cluster_uuid = model.uuid |
| 117 |
|
| 118 |
timeout = (config.Config.cluster_creation_timeout + 10) * 60 |
| 119 |
try: |
| 120 |
self.cluster_admin_client.wait_for_created_cluster( |
| 121 |
model.uuid, delete_on_error=False, timeout=timeout) |
| 122 |
except Exception as e: |
| 123 |
self.LOG.debug("Cluster create exception: %s\n" % e) |
| 124 |
self.copy_logs_handler( |
| 125 |
lambda: list( |
| 126 |
[self._get_cluster_by_id(model.uuid)[1].master_addresses, |
| 127 |
self._get_cluster_by_id(model.uuid)[1].node_addresses]), |
| 128 |
self.cluster_template.coe, |
| 129 |
self.keypair) |
| 130 |
raise |
| 131 |
|
| 132 |
if config.Config.copy_logs_success: |
| 133 |
self.LOG.debug('Copying logs on success') |
| 134 |
self.copy_logs_handler( |
| 135 |
lambda: list( |
| 136 |
[self._get_cluster_by_id(model.uuid)[1].master_addresses, |
| 137 |
self._get_cluster_by_id(model.uuid)[1].node_addresses]), |
| 138 |
self.cluster_template.coe, |
| 139 |
self.keypair) |
| 140 |
|
| 141 |
return resp, model |
| 142 |
|
| 143 |
def _delete_cluster(self, cluster_id): |
| 144 |
self.LOG.debug('We will delete a cluster for %s', cluster_id) |
| 145 |
resp, model = self.cluster_member_client.delete_cluster(cluster_id) |
| 146 |
self.assertEqual(204, resp.status) |
| 147 |
|
| 148 |
self.cluster_admin_client.wait_for_cluster_to_delete(cluster_id) |
| 149 |
|
| 150 |
self.assertRaises(exceptions.NotFound, |
| 151 |
self.cert_reader_client.get_cert, |
| 152 |
cluster_id, headers=HEADERS) |
| 153 |
return resp, model |
| 154 |
|
| 155 |
def _get_cluster_by_id(self, cluster_id): |
| 156 |
resp, model = self.cluster_reader_client.get_cluster(cluster_id) |
| 157 |
return resp, model |
| 158 |
|
| 159 |
# (dimtruck) Combining all these tests in one because |
| 160 |
# they time out on the gate (2 hours not enough) |
| 161 |
@testtools.testcase.attr('positive') |
| 162 |
@testtools.testcase.attr('slow') |
| 163 |
@decorators.idempotent_id( |
| 164 |
IDEMPOTENT_IDS['test_create_list_sign_delete_clusters']) |
| 165 |
def test_create_list_sign_delete_clusters(self): |
| 166 |
gen_model = datagen.valid_cluster_data( |
| 167 |
cluster_template_id=self.cluster_template.uuid, node_count=1) |
| 168 |
|
| 169 |
# test cluster create |
| 170 |
_, cluster_model = self._create_cluster(gen_model) |
| 171 |
self.assertNotIn('status', cluster_model) |
| 172 |
|
| 173 |
_, cluster_model = self._get_cluster_by_id(cluster_model.uuid) |
| 174 |
|
| 175 |
# test cluster list |
| 176 |
resp, cluster_list_model = self.cluster_reader_client.list_clusters() |
| 177 |
self.assertEqual(200, resp.status) |
| 178 |
self.assertGreater(len(cluster_list_model.clusters), 0) |
| 179 |
self.assertIn( |
| 180 |
cluster_model.uuid, list([x['uuid'] |
| 181 |
for x in cluster_list_model.clusters])) |
| 182 |
|
| 183 |
# test ca show |
| 184 |
resp, ca = self.cert_reader_client.get_cert( |
| 185 |
cluster_model.uuid, headers=HEADERS) |
| 186 |
self.LOG.debug("cert resp: %s", resp) |
| 187 |
self.assertEqual(200, resp.status) |
| 188 |
self.assertEqual(ca.cluster_uuid, cluster_model.uuid) |
| 189 |
self.assertIsNotNone(ca.pem) |
| 190 |
self.assertIn('-----BEGIN CERTIFICATE-----', ca.pem) |
| 191 |
self.assertIn('-----END CERTIFICATE-----', ca.pem) |
| 192 |
|
| 193 |
# test ca sign |
| 194 |
csr_sample = utils.generate_csr_and_key() |
| 195 |
cert_data_model = datagen.cert_data(cluster_model.uuid, |
| 196 |
csr_data=csr_sample['csr']) |
| 197 |
resp, cert_model = self.cert_member_client.post_cert( |
| 198 |
cert_data_model, headers=HEADERS) |
| 199 |
self.LOG.debug("cert resp: %s", resp) |
| 200 |
self.assertEqual(201, resp.status) |
| 201 |
self.assertEqual(cert_model.cluster_uuid, cluster_model.uuid) |
| 202 |
self.assertIsNotNone(cert_model.pem) |
| 203 |
self.assertIn('-----BEGIN CERTIFICATE-----', cert_model.pem) |
| 204 |
self.assertIn('-----END CERTIFICATE-----', cert_model.pem) |
| 205 |
|
| 206 |
# test Kubernetes API |
| 207 |
kube_cfg = """ |
| 208 |
--- |
| 209 |
apiVersion: v1 |
| 210 |
clusters: |
| 211 |
- cluster: |
| 212 |
certificate-authority-data: {ca} |
| 213 |
server: {api_address} |
| 214 |
name: {name} |
| 215 |
contexts: |
| 216 |
- context: |
| 217 |
cluster: {name} |
| 218 |
user: admin |
| 219 |
name: default |
| 220 |
current-context: default |
| 221 |
kind: Config |
| 222 |
preferences: {{}} |
| 223 |
users: |
| 224 |
- name: admin |
| 225 |
user: |
| 226 |
client-certificate-data: {cert} |
| 227 |
client-key-data: {key} |
| 228 |
""".format(name=cluster_model.name, |
| 229 |
api_address=cluster_model.api_address, |
| 230 |
key=base64.encode_as_text(csr_sample['private_key']), |
| 231 |
cert=base64.encode_as_text(cert_model.pem), |
| 232 |
ca=base64.encode_as_text(ca.pem)) |
| 233 |
|
| 234 |
kube_cfg = safe_load(kube_cfg) |
| 235 |
kube_config = kubernetes.config.load_kube_config_from_dict(kube_cfg) |
| 236 |
|
| 237 |
# Get nodes and pods list using kubernetes python client |
| 238 |
with kubernetes.client.ApiClient(kube_config) as api_client: |
| 239 |
v1 = kubernetes.client.CoreV1Api(api_client) |
| 240 |
try: |
| 241 |
list_nodes = v1.list_node(pretty="true") |
| 242 |
with open("/tmp/magnum-nodes/api-list-nodes", "w") as outfile: |
| 243 |
outfile.write(str(list_nodes)) |
| 244 |
list_pods = v1.list_pod_for_all_namespaces(pretty="true") |
| 245 |
with open("/tmp/magnum-nodes/api-list-pods", "w") as outfile: |
| 246 |
outfile.write(str(list_pods)) |
| 247 |
except ApiException as e: |
| 248 |
print("Exception when calling CoreV1Api: %s\n" % e) |
| 249 |
raise |
| 250 |
|
| 251 |
# Get nodes and pods using kubectl |
| 252 |
with open("/tmp/magnum-nodes/kube.conf", "w") as outfile: |
| 253 |
outfile.write(str(kube_cfg)) |
| 254 |
try: |
| 255 |
self.copy_pod_logs() |
| 256 |
except Exception as e: |
| 257 |
self.LOG.debug("kubectl exception: %s\n" % e) |
| 258 |
raise |
| 259 |
|
| 260 |
# test cluster delete |
| 261 |
self._delete_cluster(cluster_model.uuid) |
| 262 |
self.clusters.remove(cluster_model.uuid) |
| 263 |
|
| 264 |
@testtools.testcase.attr('negative') |
| 265 |
@decorators.idempotent_id( |
| 266 |
IDEMPOTENT_IDS['test_create_cluster_for_nonexisting_cluster_template']) |
| 267 |
def test_create_cluster_for_nonexisting_cluster_template(self): |
| 268 |
cm_id = 'this-does-not-exist' |
| 269 |
gen_model = datagen.valid_cluster_data(cluster_template_id=cm_id) |
| 270 |
self.assertRaises( |
| 271 |
exceptions.BadRequest, |
| 272 |
self.cluster_member_client.post_cluster, gen_model) |
| 273 |
|
| 274 |
@testtools.testcase.attr('positive') |
| 275 |
@testtools.testcase.attr('slow') |
| 276 |
@decorators.idempotent_id( |
| 277 |
IDEMPOTENT_IDS['test_create_cluster_with_zero_nodes']) |
| 278 |
def test_create_cluster_with_zero_nodes(self): |
| 279 |
gen_model = datagen.valid_cluster_data( |
| 280 |
cluster_template_id=self.cluster_template.uuid, node_count=0) |
| 281 |
|
| 282 |
# test cluster create |
| 283 |
_, cluster_model = self._create_cluster(gen_model) |
| 284 |
self.assertNotIn('status', cluster_model) |
| 285 |
|
| 286 |
# test cluster delete |
| 287 |
self._delete_cluster(cluster_model.uuid) |
| 288 |
self.clusters.remove(cluster_model.uuid) |
| 289 |
|
| 290 |
@testtools.testcase.attr('negative') |
| 291 |
@decorators.idempotent_id( |
| 292 |
IDEMPOTENT_IDS['test_create_cluster_with_zero_masters']) |
| 293 |
def test_create_cluster_with_zero_masters(self): |
| 294 |
uuid = self.cluster_template.uuid |
| 295 |
gen_model = datagen.valid_cluster_data(cluster_template_id=uuid, |
| 296 |
master_count=0) |
| 297 |
self.assertRaises( |
| 298 |
exceptions.BadRequest, |
| 299 |
self.cluster_member_client.post_cluster, gen_model) |
| 300 |
|
| 301 |
@testtools.testcase.attr('negative') |
| 302 |
@decorators.idempotent_id( |
| 303 |
IDEMPOTENT_IDS['test_create_cluster_with_nonexisting_flavor']) |
| 304 |
def test_create_cluster_with_nonexisting_flavor(self): |
| 305 |
gen_model = \ |
| 306 |
datagen.cluster_template_data_with_valid_keypair_image_flavor() |
| 307 |
resp, cluster_template = self._create_cluster_template(gen_model) |
| 308 |
self.assertEqual(201, resp.status) |
| 309 |
self.assertIsNotNone(cluster_template.uuid) |
| 310 |
|
| 311 |
uuid = cluster_template.uuid |
| 312 |
gen_model = datagen.valid_cluster_data(cluster_template_id=uuid) |
| 313 |
gen_model.flavor_id = 'aaa' |
| 314 |
self.assertRaises(exceptions.BadRequest, |
| 315 |
self.cluster_member_client.post_cluster, gen_model) |
| 316 |
|
| 317 |
resp, _ = self._delete_cluster_template(cluster_template.uuid) |
| 318 |
self.assertEqual(204, resp.status) |
| 319 |
|
| 320 |
@testtools.testcase.attr('negative') |
| 321 |
@decorators.idempotent_id( |
| 322 |
IDEMPOTENT_IDS['test_update_cluster_for_nonexisting_cluster']) |
| 323 |
def test_update_cluster_for_nonexisting_cluster(self): |
| 324 |
patch_model = datagen.cluster_name_patch_data() |
| 325 |
|
| 326 |
self.assertRaises( |
| 327 |
exceptions.NotFound, |
| 328 |
self.cluster_member_client.patch_cluster, 'fooo', patch_model) |
| 329 |
|
| 330 |
@testtools.testcase.attr('negative') |
| 331 |
@decorators.idempotent_id( |
| 332 |
IDEMPOTENT_IDS['test_delete_cluster_for_nonexisting_cluster']) |
| 333 |
def test_delete_cluster_for_nonexisting_cluster(self): |
| 334 |
self.assertRaises( |
| 335 |
exceptions.NotFound, |
| 336 |
self.cluster_member_client.delete_cluster, data_utils.rand_uuid()) |