Debian Codegraph @

ibm_cloud_sdk_core/base_service.py

raw
1 # coding: utf-8
2
3 # Copyright 2019, 2024 IBM All Rights Reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16
17 import gzip
18 import io
19 import logging
20 import json as json_import
21 from http.cookiejar import CookieJar
22 from http import client
23 from os.path import basename
24 from typing import Dict, List, Optional, Tuple, Union
25 from urllib3.util.retry import Retry
26
27 import requests
28 from requests.structures import CaseInsensitiveDict
29 from requests.exceptions import JSONDecodeError
30
31 from ibm_cloud_sdk_core.authenticators import Authenticator
32 from .api_exception import ApiException
33 from .detailed_response import DetailedResponse
34 from .http_adapter import SSLHTTPAdapter
35 from .token_managers.token_manager import TokenManager
36 from .utils import (
37 has_bad_first_or_last_char,
38 is_json_mimetype,
39 remove_null_values,
40 cleanup_values,
41 read_external_sources,
42 strip_extra_slashes,
43 GzipStream,
44 )
45 from .private_helpers import _build_user_agent
46 from .logger import (
47 get_logger,
48 LoggingFilter,
49 )
50
51 logger = get_logger()
52
53
54 # pylint: disable=too-many-instance-attributes
55 # pylint: disable=too-many-locals
56 class BaseService:
57 """Common functionality shared by generated service classes.
58
59 The base service authenticates requests via its authenticator, stores cookies, and
60 wraps responses from the service endpoint in DetailedResponse or APIException objects.
61
62 Keyword Arguments:
63 service_url: Url to the service endpoint. Defaults to None.
64 authenticator: Adds authentication data to service requests. Defaults to None.
65 disable_ssl_verification: A flag that indicates whether verification of the server's SSL
66 certificate should be disabled or not. Defaults to False.
67 enable_gzip_compression: A flag that indicates whether to enable gzip compression on request bodies
68
69 Attributes:
70 service_url (str): Url to the service endpoint.
71 authenticator (Authenticator): Adds authentication data to service requests.
72 disable_ssl_verification (bool): A flag that indicates whether verification of
73 the server's SSL certificate should be disabled or not.
74 default_headers (dict): A dictionary of headers to be sent with every HTTP request to the service endpoint.
75 jar (http.cookiejar.CookieJar): Stores cookies received from the service.
76 http_config (dict): A dictionary containing values that control the timeout, proxies, and etc of HTTP requests.
77 http_client (Session): A configurable session which can use Transport Adapters to configure retries, timeouts,
78 proxies, etc. globally for all requests.
79 enable_gzip_compression (bool): A flag that indicates whether to enable gzip compression on request bodies
80 Raises:
81 ValueError: If Authenticator is not provided or invalid type.
82 """
83
84 ERROR_MSG_DISABLE_SSL = (
85 'The connection failed because the SSL certificate is not valid. To use a self-signed '
86 'certificate, disable verification of the server\'s SSL certificate by invoking the '
87 'set_disable_ssl_verification(True) on your service instance and/ or use the '
88 'disable_ssl_verification option of the authenticator.'
89 )
90
91 def __init__(
92 self,
93 *,
94 service_url: str = None,
95 authenticator: Optional[Authenticator] = None,
96 disable_ssl_verification: bool = False,
97 enable_gzip_compression: bool = False,
98 ) -> None:
99 self.set_service_url(service_url)
100 self.http_client = requests.Session()
101 self.http_config = {}
102 self.jar = CookieJar()
103 self.authenticator = authenticator
104 self.disable_ssl_verification = disable_ssl_verification
105 self.default_headers = None
106 self.enable_gzip_compression = enable_gzip_compression
107 self._set_user_agent_header(_build_user_agent())
108 self.retry_config = None
109 self.http_adapter = SSLHTTPAdapter(_disable_ssl_verification=self.disable_ssl_verification)
110 if not self.authenticator:
111 raise ValueError('authenticator must be provided')
112 if not isinstance(self.authenticator, Authenticator):
113 raise ValueError('authenticator should be of type Authenticator')
114
115 self.http_client.mount('http://', self.http_adapter)
116 self.http_client.mount('https://', self.http_adapter)
117 # If debug logging is requested, then trigger HTTP message logging as well.
118 if logger.isEnabledFor(logging.DEBUG):
119 client.HTTPConnection.debuglevel = 1
120 # Replace the `print` function in the HTTPClient module to
121 # use the debug logger instead of the bare Python print.
122 client.print = lambda *args: logger.debug(LoggingFilter.filter_message(" ".join(args)))
123
124 def enable_retries(self, max_retries: int = 4, retry_interval: float = 30.0) -> None:
125 """Enable automatic retries on the underlying http client used by the BaseService instance.
126
127 Args:
128 max_retries: the maximum number of retries to attempt for a failed retryable request
129 retry_interval: the maximum wait time (in seconds) to use for retry attempts.
130 In general, if a response includes the Retry-After header, that will be used for
131 the wait time associated with the retry attempt. If the Retry-After header is not
132 present, then the wait time is based on an exponential backoff policy with a maximum
133 backoff time of "retry_interval".
134 """
135 self.retry_config = Retry(
136 total=max_retries,
137 backoff_factor=1.0,
138 backoff_max=retry_interval,
139 # List of HTTP status codes to retry on in addition to Timeout/Connection Errors
140 status_forcelist=[429, 500, 502, 503, 504],
141 # List of HTTP methods to retry on
142 # Omitting this will default to all methods except POST
143 allowed_methods=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'],
144 )
145 self.http_adapter = SSLHTTPAdapter(
146 max_retries=self.retry_config, _disable_ssl_verification=self.disable_ssl_verification
147 )
148 self.http_client.mount('http://', self.http_adapter)
149 self.http_client.mount('https://', self.http_adapter)
150 logger.debug('Enabled retries; max_retries=%d, max_retry_interval=%f', max_retries, retry_interval)
151
152 def disable_retries(self):
153 """Remove retry config from http_adapter"""
154 self.retry_config = None
155 self.http_adapter = SSLHTTPAdapter(_disable_ssl_verification=self.disable_ssl_verification)
156 self.http_client.mount('http://', self.http_adapter)
157 self.http_client.mount('https://', self.http_adapter)
158 logger.debug('Disabled retries')
159
160 def configure_service(self, service_name: str) -> None:
161 """Look for external configuration of a service. Set service properties.
162
163 Try to get config from external sources, with the following priority:
164 1. Credentials file(ibm-credentials.env)
165 2. Environment variables
166 3. VCAP Services(Cloud Foundry)
167
168 Args:
169 service_name: The service name
170
171 Raises:
172 ValueError: If service_name is not a string.
173 """
174 if not isinstance(service_name, str):
175 raise ValueError('Service_name must be of type string.')
176
177 logger.debug('Configuring BaseService instance with service name: %s', service_name)
178
179 config = read_external_sources(service_name)
180 if config.get('URL'):
181 self.set_service_url(config.get('URL'))
182 if config.get('DISABLE_SSL'):
183 self.set_disable_ssl_verification(config.get('DISABLE_SSL').lower() == 'true')
184 if config.get('ENABLE_GZIP'):
185 self.set_enable_gzip_compression(config.get('ENABLE_GZIP').lower() == 'true')
186 if config.get('ENABLE_RETRIES'):
187 if config.get('ENABLE_RETRIES').lower() == 'true':
188 kwargs = {}
189 if config.get('MAX_RETRIES'):
190 kwargs["max_retries"] = int(config.get('MAX_RETRIES'))
191 if config.get('RETRY_INTERVAL'):
192 kwargs["retry_interval"] = float(config.get('RETRY_INTERVAL'))
193 self.enable_retries(**kwargs)
194
195 def _set_user_agent_header(self, user_agent_string: str) -> None:
196 self.user_agent_header = {'User-Agent': user_agent_string}
197 logger.debug('Set User-Agent: %s', user_agent_string)
198
199 def set_http_config(self, http_config: dict) -> None:
200 """Sets the http config dictionary.
201
202 The dictionary can contain values that control the timeout, proxies, and etc of HTTP requests.
203
204 Arguments:
205 http_config: Configuration values to customize HTTP behaviors.
206
207 Raises:
208 TypeError: http_config is not a dict.
209 """
210 if isinstance(http_config, dict):
211 self.http_config = http_config
212 if (
213 self.authenticator
214 and hasattr(self.authenticator, 'token_manager')
215 and isinstance(self.authenticator.token_manager, TokenManager)
216 ):
217 self.authenticator.token_manager.http_config = http_config
218 else:
219 raise TypeError("http_config parameter must be a dictionary")
220
221 def set_disable_ssl_verification(self, status: bool = False) -> None:
222 """Set the flag that indicates whether verification of
223 the server's SSL certificate should be disabled or not.
224
225 Keyword Arguments:
226 status: set to true to disable ssl verification (default: {False})
227 """
228 if self.disable_ssl_verification == status:
229 # Do nothing if the state doesn't change.
230 return
231
232 self.disable_ssl_verification = status
233
234 self.http_adapter = SSLHTTPAdapter(
235 max_retries=self.retry_config, _disable_ssl_verification=self.disable_ssl_verification
236 )
237 self.http_client.mount('http://', self.http_adapter)
238 self.http_client.mount('https://', self.http_adapter)
239 logger.debug('Disabled SSL verification in HTTP client')
240
241 def set_service_url(self, service_url: str) -> None:
242 """Set the url the service will make HTTP requests too.
243
244 Arguments:
245 service_url: The WHATWG URL standard origin ex. https://example.service.com
246
247 Raises:
248 ValueError: Improperly formatted service_url
249 """
250 if has_bad_first_or_last_char(service_url):
251 raise ValueError(
252 'The service url shouldn\'t start or end with curly brackets or quotes. '
253 'Be sure to remove any {} and \" characters surrounding your service url'
254 )
255 if service_url is not None:
256 service_url = service_url.rstrip('/')
257 self.service_url = service_url
258 logger.debug('Set service URL: %s', service_url)
259
260 def get_http_client(self) -> requests.sessions.Session:
261 """Get the http client session currently used by the service.
262
263 Returns:
264 The http client session currently used by the service.
265 """
266 return self.http_client
267
268 def set_http_client(self, http_client: requests.sessions.Session) -> None:
269 """Set current http client session
270
271 Arguments:
272 http_client: A new requests session client
273 """
274 if isinstance(http_client, requests.sessions.Session):
275 self.http_client = http_client
276 else:
277 raise TypeError("http_client parameter must be a requests.sessions.Session")
278
279 def get_authenticator(self) -> Authenticator:
280 """Get the authenticator currently used by the service.
281
282 Returns:
283 The authenticator currently used by the service.
284 """
285 return self.authenticator
286
287 def set_default_headers(self, headers: Dict[str, str]) -> None:
288 """Set http headers to be sent in every request.
289
290 Arguments:
291 headers: A dictionary of headers
292 """
293 if isinstance(headers, dict):
294 self.default_headers = headers
295 else:
296 raise TypeError("headers parameter must be a dictionary")
297
298 def send(self, request: requests.Request, **kwargs) -> DetailedResponse:
299 """Send a request and wrap the response in a DetailedResponse or APIException.
300
301 Args:
302 request: The request to send to the service endpoint.
303
304 Raises:
305 ApiException: The exception from the API.
306
307 Returns:
308 The response from the request.
309 """
310 # Use a one minute timeout when our caller doesn't give a timeout.
311 # http://docs.python-requests.org/en/master/user/quickstart/#timeouts
312 kwargs = dict({"timeout": 60}, **kwargs)
313 kwargs = dict(kwargs, **self.http_config)
314
315 if self.disable_ssl_verification:
316 kwargs['verify'] = False
317
318 # Check to see if the caller specified the 'stream' argument.
319 stream_response = kwargs.get('stream') or False
320
321 # Remove the keys we set manually, don't let the user overwrite these.
322 reserved_keys = ['method', 'url', 'headers', 'params', 'cookies']
323 silent_keys = ['headers']
324 for key in reserved_keys:
325 if key in kwargs:
326 del kwargs[key]
327 if key not in silent_keys:
328 logger.warning('"%s" has been removed from the request', key)
329 try:
330 logger.debug('Sending HTTP request message')
331
332 response = self.http_client.request(**request, cookies=self.jar, **kwargs)
333
334 logger.debug('Received HTTP response message, status code %d', response.status_code)
335
336 # Process a "success" response.
337 if 200 <= response.status_code <= 299:
338 if response.status_code == 204 or request['method'] == 'HEAD':
339 # There is no body content for a HEAD response or a 204 response.
340 result = None
341 elif stream_response:
342 result = response
343 elif not response.text:
344 result = None
345 elif is_json_mimetype(response.headers.get('Content-Type')):
346 # If this is a JSON response, then try to unmarshal it.
347 try:
348 result = response.json(strict=False)
349 except JSONDecodeError as err:
350 raise ApiException(
351 code=response.status_code,
352 http_response=response,
353 message='Error processing the HTTP response',
354 ) from err
355 else:
356 # Non-JSON response, just use response body as-is.
357 result = response
358
359 return DetailedResponse(response=result, headers=response.headers, status_code=response.status_code)
360
361 # Received error status code from server, raise an APIException.
362 raise ApiException(response.status_code, http_response=response)
363 except requests.exceptions.SSLError:
364 logger.exception(self.ERROR_MSG_DISABLE_SSL)
365 raise
366
367 def set_enable_gzip_compression(self, should_enable_compression: bool = False) -> None:
368 """Set value to enable gzip compression on request bodies"""
369 self.enable_gzip_compression = should_enable_compression
370
371 def get_enable_gzip_compression(self) -> bool:
372 """Get value for enabling gzip compression on request bodies"""
373 return self.enable_gzip_compression
374
375 def prepare_request(
376 self,
377 method: str,
378 url: str,
379 *,
380 headers: Optional[dict] = None,
381 params: Optional[dict] = None,
382 data: Optional[Union[str, dict]] = None,
383 files: Optional[Union[Dict[str, Tuple[str]], List[Tuple[str, Tuple[str, ...]]]]] = None,
384 **kwargs,
385 ) -> dict:
386 """Build a dict that represents an HTTP service request.
387
388 Clean up headers, add default http configuration, convert data
389 into json, process files, and merge all into a single request dict.
390
391 Args:
392 method: The HTTP method of the request ex. GET, POST, etc.
393 url: The origin + pathname according to WHATWG spec.
394
395 Keyword Arguments:
396 headers: A dictionary containing the headers to be included in the request.
397 Entries with a value of None will be ignored (excluded).
398 params: A dictionary containing the query parameters to be included in the request.
399 Entries with a value of None will be ignored (excluded).
400 data: The request body. Converted to json if a dict.
401 files: 'files' can be a dictionary (i.e { '<part-name>': (<tuple>)}),
402 or a list of tuples [ (<part-name>, (<tuple>))... ]
403
404 Returns:
405 Prepared request dictionary.
406 """
407 # pylint: disable=unused-argument; necessary for kwargs
408 request = {'method': method}
409
410 # validate the service url is set
411 if not self.service_url:
412 raise ValueError('The service_url is required')
413
414 # Combine the service_url and operation path to form the request url.
415 # Note: we have already stripped any trailing slashes from the service_url
416 # and we know that the operation path ('url') will start with a slash.
417 request['url'] = strip_extra_slashes(self.service_url + url)
418
419 headers = remove_null_values(headers) if headers else {}
420 headers = cleanup_values(headers)
421 headers = CaseInsensitiveDict(headers)
422 if self.default_headers is not None:
423 headers.update(self.default_headers)
424 if 'user-agent' not in headers:
425 headers.update(self.user_agent_header)
426 request['headers'] = headers
427
428 params = remove_null_values(params)
429 params = cleanup_values(params)
430 request['params'] = params
431
432 if isinstance(data, str):
433 data = data.encode('utf-8')
434 elif isinstance(data, dict) and data:
435 data = remove_null_values(data)
436 if headers.get('content-type') is None:
437 headers.update({'content-type': 'application/json'})
438 data = json_import.dumps(data).encode('utf-8')
439 request['data'] = data
440
441 self.authenticator.authenticate(request)
442
443 # Compress the request body if applicable
444 if self.get_enable_gzip_compression() and 'content-encoding' not in headers and request['data'] is not None:
445 headers['content-encoding'] = 'gzip'
446 request['headers'] = headers
447 # If the provided data is a file-like object, we create `GzipStream` which will handle
448 # the compression on-the-fly when the requests package starts reading its content.
449 # This helps avoid OOM errors when the opened file is too big.
450 # In any other cases, we use the in memory compression directly from
451 # the `gzip` package for backward compatibility.
452 raw_data = request['data']
453 request['data'] = GzipStream(raw_data) if isinstance(raw_data, io.IOBase) else gzip.compress(raw_data)
454
455 # Next, we need to process the 'files' argument to try to fill in
456 # any missing filenames where possible.
457 # 'files' can be a dictionary (i.e { '<part-name>': (<tuple>)} )
458 # or a list of tuples [ (<part-name>, (<tuple>))... ]
459 # If 'files' is a dictionary we'll convert it to a list of tuples.
460 new_files = []
461 if files is not None:
462 # If 'files' is a dictionary, transform it into a list of tuples.
463 if isinstance(files, dict):
464 files = remove_null_values(files)
465 files = files.items()
466 # Next, fill in any missing filenames from file tuples.
467 for part_name, file_tuple in files:
468 if file_tuple and len(file_tuple) == 3 and file_tuple[0] is None:
469 file = file_tuple[1]
470 if file and hasattr(file, 'name'):
471 filename = basename(file.name)
472 file_tuple = (filename, file_tuple[1], file_tuple[2])
473 new_files.append((part_name, file_tuple))
474 request['files'] = new_files
475 logger.debug('Prepared request [%s %s]', request['method'], request['url'])
476 return request
477
478 @staticmethod
479 def encode_path_vars(*args: str) -> List[str]:
480 """Encode path variables to be substituted into a URL path.
481
482 Arguments:
483 args: A list of strings to be URL path encoded
484
485 Returns:
486 A list of encoded strings that are safe to substitute into a URL path.
487 """
488 return (requests.utils.quote(x, safe='') for x in args)
489
490 # The methods below are kept for compatibility and should be removed
491 # in the next major release.
492
493 # pylint: disable=protected-access
494
495 @staticmethod
496 def _convert_model(val: str) -> None:
497 if isinstance(val, str):
498 val = json_import.loads(val)
499 if hasattr(val, "_to_dict"):
500 return val._to_dict()
501 return val
502
503 @staticmethod
504 def _convert_list(val: list) -> None:
505 if isinstance(val, list):
506 return ",".join(val)
507 return val
508
509 @staticmethod
510 def _encode_path_vars(*args) -> None:
511 return BaseService.encode_path_vars(*args)