| 1 |
# This code is part of Ansible, but is an independent component. |
| 2 |
# This particular file snippet, and this file snippet only, is BSD licensed. |
| 3 |
# Modules you write using this snippet, which is embedded dynamically by Ansible |
| 4 |
# still belong to the author of the module, and may assign their own license |
| 5 |
# to the complete work. |
| 6 |
# |
| 7 |
# Copyright (c) 2017-2021, NetApp Ansible Team <[email protected]> |
| 8 |
# All rights reserved. |
| 9 |
# |
| 10 |
# Redistribution and use in source and binary forms, with or without modification, |
| 11 |
# are permitted provided that the following conditions are met: |
| 12 |
# |
| 13 |
# * Redistributions of source code must retain the above copyright |
| 14 |
# notice, this list of conditions and the following disclaimer. |
| 15 |
# * Redistributions in binary form must reproduce the above copyright notice, |
| 16 |
# this list of conditions and the following disclaimer in the documentation |
| 17 |
# and/or other materials provided with the distribution. |
| 18 |
# |
| 19 |
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 20 |
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 21 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| 22 |
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
| 23 |
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| 24 |
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 25 |
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 26 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE |
| 27 |
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 |
|
| 29 |
""" |
| 30 |
netapp.py: wrapper around send_requests and other utilities |
| 31 |
""" |
| 32 |
|
| 33 |
from __future__ import (absolute_import, division, print_function) |
| 34 |
__metaclass__ = type |
| 35 |
|
| 36 |
import logging |
| 37 |
import time |
| 38 |
from ansible.module_utils.basic import missing_required_lib |
| 39 |
|
| 40 |
try: |
| 41 |
from ansible.module_utils.ansible_release import __version__ as ansible_version |
| 42 |
except ImportError: |
| 43 |
ansible_version = 'unknown' |
| 44 |
|
| 45 |
COLLECTION_VERSION = "21.24.0" |
| 46 |
PROD_ENVIRONMENT = { |
| 47 |
'CLOUD_MANAGER_HOST': 'cloudmanager.cloud.netapp.com', |
| 48 |
'AUTH0_DOMAIN': 'netapp-cloud-account.auth0.com', |
| 49 |
'SA_AUTH_HOST': 'cloudmanager.cloud.netapp.com/auth/oauth/token', |
| 50 |
'AUTH0_CLIENT': 'Mu0V1ywgYteI6w1MbD15fKfVIUrNXGWC', |
| 51 |
'AMI_FILTER': 'Setup-As-Service-AMI-Prod*', |
| 52 |
'AWS_ACCOUNT': '952013314444', |
| 53 |
'GCP_IMAGE_PROJECT': 'netapp-cloudmanager', |
| 54 |
'GCP_IMAGE_FAMILY': 'cloudmanager', |
| 55 |
'CVS_HOST_NAME': 'https://api.services.cloud.netapp.com' |
| 56 |
} |
| 57 |
STAGE_ENVIRONMENT = { |
| 58 |
'CLOUD_MANAGER_HOST': 'staging.cloudmanager.cloud.netapp.com', |
| 59 |
'AUTH0_DOMAIN': 'staging-netapp-cloud-account.auth0.com', |
| 60 |
'SA_AUTH_HOST': 'staging.cloudmanager.cloud.netapp.com/auth/oauth/token', |
| 61 |
'AUTH0_CLIENT': 'O6AHa7kedZfzHaxN80dnrIcuPBGEUvEv', |
| 62 |
'AMI_FILTER': 'Setup-As-Service-AMI-*', |
| 63 |
'AWS_ACCOUNT': '282316784512', |
| 64 |
'GCP_IMAGE_PROJECT': 'tlv-automation', |
| 65 |
'GCP_IMAGE_FAMILY': 'occm-automation', |
| 66 |
'CVS_HOST_NAME': 'https://staging.api.services.cloud.netapp.com' |
| 67 |
} |
| 68 |
|
| 69 |
try: |
| 70 |
import requests |
| 71 |
HAS_REQUESTS = True |
| 72 |
except ImportError: |
| 73 |
HAS_REQUESTS = False |
| 74 |
|
| 75 |
|
| 76 |
POW2_BYTE_MAP = dict( |
| 77 |
# Here, 1 kb = 1024 |
| 78 |
bytes=1, |
| 79 |
b=1, |
| 80 |
kb=1024, |
| 81 |
mb=1024 ** 2, |
| 82 |
gb=1024 ** 3, |
| 83 |
tb=1024 ** 4, |
| 84 |
pb=1024 ** 5, |
| 85 |
eb=1024 ** 6, |
| 86 |
zb=1024 ** 7, |
| 87 |
yb=1024 ** 8 |
| 88 |
) |
| 89 |
|
| 90 |
|
| 91 |
LOG = logging.getLogger(__name__) |
| 92 |
LOG_FILE = '/tmp/cloudmanager_apis.log' |
| 93 |
|
| 94 |
|
| 95 |
def cloudmanager_host_argument_spec(): |
| 96 |
|
| 97 |
return dict( |
| 98 |
refresh_token=dict(required=False, type='str', no_log=True), |
| 99 |
sa_client_id=dict(required=False, type='str', no_log=True), |
| 100 |
sa_secret_key=dict(required=False, type='str', no_log=True), |
| 101 |
environment=dict(required=False, type='str', choices=['prod', 'stage'], default='prod'), |
| 102 |
feature_flags=dict(required=False, type='dict') |
| 103 |
) |
| 104 |
|
| 105 |
|
| 106 |
def has_feature(module, feature_name): |
| 107 |
feature = get_feature(module, feature_name) |
| 108 |
if isinstance(feature, bool): |
| 109 |
return feature |
| 110 |
module.fail_json(msg="Error: expected bool type for feature flag: %s, found %s" % (feature_name, type(feature))) |
| 111 |
|
| 112 |
|
| 113 |
def get_feature(module, feature_name): |
| 114 |
''' if the user has configured the feature, use it |
| 115 |
otherwise, use our default |
| 116 |
''' |
| 117 |
default_flags = dict( |
| 118 |
trace_apis=False, # if True, append REST requests/responses to /tmp/cloudmanager_apis.log |
| 119 |
trace_headers=False, # if True, and if trace_apis is True, include <large> headers in trace |
| 120 |
show_modified=True, |
| 121 |
simulator=False, # if True, it is running on simulator |
| 122 |
) |
| 123 |
|
| 124 |
if module.params['feature_flags'] is not None and feature_name in module.params['feature_flags']: |
| 125 |
return module.params['feature_flags'][feature_name] |
| 126 |
if feature_name in default_flags: |
| 127 |
return default_flags[feature_name] |
| 128 |
module.fail_json(msg="Internal error: unexpected feature flag: %s" % feature_name) |
| 129 |
|
| 130 |
|
| 131 |
class CloudManagerRestAPI(object): |
| 132 |
""" wrapper around send_request """ |
| 133 |
def __init__(self, module, timeout=60): |
| 134 |
self.module = module |
| 135 |
self.timeout = timeout |
| 136 |
self.refresh_token = self.module.params['refresh_token'] |
| 137 |
self.sa_client_id = self.module.params['sa_client_id'] |
| 138 |
self.sa_secret_key = self.module.params['sa_secret_key'] |
| 139 |
self.environment = self.module.params['environment'] |
| 140 |
if self.environment == 'prod': |
| 141 |
self.environment_data = PROD_ENVIRONMENT |
| 142 |
elif self.environment == 'stage': |
| 143 |
self.environment_data = STAGE_ENVIRONMENT |
| 144 |
self.url = 'https://' |
| 145 |
self.api_root_path = None |
| 146 |
self.check_required_library() |
| 147 |
if has_feature(module, 'trace_apis'): |
| 148 |
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s') |
| 149 |
self.log_headers = has_feature(module, 'trace_headers') # requires trace_apis to do anything |
| 150 |
self.simulator = has_feature(module, 'simulator') |
| 151 |
self.token_type, self.token = self.get_token() |
| 152 |
|
| 153 |
def check_required_library(self): |
| 154 |
if not HAS_REQUESTS: |
| 155 |
self.module.fail_json(msg=missing_required_lib('requests')) |
| 156 |
|
| 157 |
def format_client_id(self, client_id): |
| 158 |
return client_id if client_id.endswith('clients') else client_id + 'clients' |
| 159 |
|
| 160 |
def build_url(self, api): |
| 161 |
# most requests are sent to Cloud Manager, but for connectors we need to manage VM instances using AWS, Azure, or GCP APIs |
| 162 |
if api.startswith('http'): |
| 163 |
return api |
| 164 |
# add host if API starts with / and host is not already included in self.url |
| 165 |
prefix = self.environment_data['CLOUD_MANAGER_HOST'] if self.environment_data['CLOUD_MANAGER_HOST'] not in self.url and api.startswith('/') else '' |
| 166 |
return self.url + prefix + api |
| 167 |
|
| 168 |
def send_request(self, method, api, params, json=None, data=None, header=None, authorized=True): |
| 169 |
''' send http request and process response, including error conditions ''' |
| 170 |
url = self.build_url(api) |
| 171 |
headers = { |
| 172 |
'Content-type': "application/json", |
| 173 |
'Referer': "Ansible_NetApp", |
| 174 |
} |
| 175 |
if authorized: |
| 176 |
headers['Authorization'] = self.token_type + " " + self.token |
| 177 |
if header is not None: |
| 178 |
headers.update(header) |
| 179 |
for __ in range(3): |
| 180 |
json_dict, error_details, on_cloud_request_id = self._send_request(method, url, params, json, data, headers) |
| 181 |
# we observe this error with DELETE on agents-mgmt/agent (and sometimes on GET) |
| 182 |
if error_details is not None and 'Max retries exceeded with url:' in error_details: |
| 183 |
time.sleep(5) |
| 184 |
else: |
| 185 |
break |
| 186 |
return json_dict, error_details, on_cloud_request_id |
| 187 |
|
| 188 |
def _send_request(self, method, url, params, json, data, headers): |
| 189 |
json_dict = None |
| 190 |
json_error = None |
| 191 |
error_details = None |
| 192 |
on_cloud_request_id = None |
| 193 |
response = None |
| 194 |
status_code = None |
| 195 |
|
| 196 |
def get_json(response): |
| 197 |
''' extract json, and error message if present ''' |
| 198 |
error = None |
| 199 |
try: |
| 200 |
json = response.json() |
| 201 |
except ValueError: |
| 202 |
return None, None |
| 203 |
success_code = [200, 201, 202] |
| 204 |
if response.status_code not in success_code: |
| 205 |
error = json.get('message') |
| 206 |
self.log_error(response.status_code, 'HTTP error: %s' % error) |
| 207 |
return json, error |
| 208 |
|
| 209 |
self.log_request(method=method, url=url, params=params, json=json, data=data, headers=headers) |
| 210 |
try: |
| 211 |
response = requests.request(method, url, headers=headers, timeout=self.timeout, params=params, json=json, data=data) |
| 212 |
status_code = response.status_code |
| 213 |
if status_code >= 300 or status_code < 200: |
| 214 |
self.log_error(status_code, 'HTTP status code error: %s' % response.content) |
| 215 |
return response.content, str(status_code), on_cloud_request_id |
| 216 |
# If the response was successful, no Exception will be raised |
| 217 |
json_dict, json_error = get_json(response) |
| 218 |
if response.headers.get('OnCloud-Request-Id', '') != '': |
| 219 |
on_cloud_request_id = response.headers.get('OnCloud-Request-Id') |
| 220 |
except requests.exceptions.HTTPError as err: |
| 221 |
self.log_error(status_code, 'HTTP error: %s' % err) |
| 222 |
error_details = str(err) |
| 223 |
except requests.exceptions.ConnectionError as err: |
| 224 |
self.log_error(status_code, 'Connection error: %s' % err) |
| 225 |
error_details = str(err) |
| 226 |
except Exception as err: |
| 227 |
self.log_error(status_code, 'Other error: %s' % err) |
| 228 |
error_details = str(err) |
| 229 |
if json_error is not None: |
| 230 |
self.log_error(status_code, 'Endpoint error: %d: %s' % (status_code, json_error)) |
| 231 |
error_details = json_error |
| 232 |
if response: |
| 233 |
self.log_debug(status_code, response.content) |
| 234 |
return json_dict, error_details, on_cloud_request_id |
| 235 |
|
| 236 |
# If an error was reported in the json payload, it is handled below |
| 237 |
def get(self, api, params=None, header=None): |
| 238 |
method = 'GET' |
| 239 |
return self.send_request(method=method, api=api, params=params, json=None, header=header) |
| 240 |
|
| 241 |
def post(self, api, data, params=None, header=None, gcp_type=False, authorized=True): |
| 242 |
method = 'POST' |
| 243 |
if gcp_type: |
| 244 |
return self.send_request(method=method, api=api, params=params, data=data, header=header) |
| 245 |
else: |
| 246 |
return self.send_request(method=method, api=api, params=params, json=data, header=header, authorized=authorized) |
| 247 |
|
| 248 |
def patch(self, api, data, params=None, header=None): |
| 249 |
method = 'PATCH' |
| 250 |
return self.send_request(method=method, api=api, params=params, json=data, header=header) |
| 251 |
|
| 252 |
def put(self, api, data, params=None, header=None): |
| 253 |
method = 'PUT' |
| 254 |
return self.send_request(method=method, api=api, params=params, json=data, header=header) |
| 255 |
|
| 256 |
def delete(self, api, data, params=None, header=None): |
| 257 |
method = 'DELETE' |
| 258 |
return self.send_request(method=method, api=api, params=params, json=data, header=header) |
| 259 |
|
| 260 |
def get_token(self): |
| 261 |
if self.sa_client_id is not None and self.sa_client_id != "" and self.sa_secret_key is not None and self.sa_secret_key != "": |
| 262 |
response, error, ocr_id = self.post(self.environment_data['SA_AUTH_HOST'], |
| 263 |
data={"grant_type": "client_credentials", "client_secret": self.sa_secret_key, |
| 264 |
"client_id": self.sa_client_id, "audience": "https://api.cloud.netapp.com"}, |
| 265 |
authorized=False) |
| 266 |
elif self.refresh_token is not None and self.refresh_token != "": |
| 267 |
response, error, ocr_id = self.post(self.environment_data['AUTH0_DOMAIN'] + '/oauth/token', |
| 268 |
data={"grant_type": "refresh_token", "refresh_token": self.refresh_token, |
| 269 |
"client_id": self.environment_data['AUTH0_CLIENT'], |
| 270 |
"audience": "https://api.cloud.netapp.com"}, |
| 271 |
authorized=False) |
| 272 |
else: |
| 273 |
self.module.fail_json(msg='Missing refresh_token or sa_client_id and sa_secret_key') |
| 274 |
|
| 275 |
if error: |
| 276 |
self.module.fail_json(msg='Error acquiring token: %s, %s' % (str(error), str(response))) |
| 277 |
token = response['access_token'] |
| 278 |
token_type = response['token_type'] |
| 279 |
|
| 280 |
return token_type, token |
| 281 |
|
| 282 |
def wait_on_completion(self, api_url, action_name, task, retries, wait_interval): |
| 283 |
while True: |
| 284 |
cvo_status, failure_error_message, error = self.check_task_status(api_url) |
| 285 |
if error is not None: |
| 286 |
return error |
| 287 |
if cvo_status == -1: |
| 288 |
return 'Failed to %s %s, error: %s' % (task, action_name, failure_error_message) |
| 289 |
elif cvo_status == 1: |
| 290 |
return None # success |
| 291 |
# status value 0 means pending |
| 292 |
if retries == 0: |
| 293 |
return 'Taking too long for %s to %s or not properly setup' % (action_name, task) |
| 294 |
time.sleep(wait_interval) |
| 295 |
retries = retries - 1 |
| 296 |
|
| 297 |
def check_task_status(self, api_url): |
| 298 |
headers = { |
| 299 |
'X-Agent-Id': self.format_client_id(self.module.params['client_id']) |
| 300 |
} |
| 301 |
|
| 302 |
network_retries = 3 |
| 303 |
while True: |
| 304 |
result, error, dummy = self.get(api_url, None, header=headers) |
| 305 |
if error is not None: |
| 306 |
if network_retries <= 0: |
| 307 |
return 0, '', error |
| 308 |
time.sleep(1) |
| 309 |
network_retries -= 1 |
| 310 |
else: |
| 311 |
response = result |
| 312 |
break |
| 313 |
return response['status'], response['error'], None |
| 314 |
|
| 315 |
def log_error(self, status_code, message): |
| 316 |
LOG.error("%s: %s", status_code, message) |
| 317 |
|
| 318 |
def log_debug(self, status_code, content): |
| 319 |
LOG.debug("%s: %s", status_code, content) |
| 320 |
|
| 321 |
def log_request(self, method, params, url, json, data, headers): |
| 322 |
contents = { |
| 323 |
'method': method, |
| 324 |
'url': url, |
| 325 |
'json': json, |
| 326 |
'data': data |
| 327 |
} |
| 328 |
if params: |
| 329 |
contents['params'] = params |
| 330 |
if self.log_headers: |
| 331 |
contents['headers'] = headers |
| 332 |
self.log_debug('sending', repr(contents)) |