Debian Codegraph @

ibm_cloud_sdk_core/utils.py

raw
1 # coding: utf-8
2
3 # Copyright 2019, 2025 IBM All Rights Reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 # from ibm_cloud_sdk_core.authenticators import Authenticator
17 import datetime
18 import gzip
19 import io
20 import json as json_import
21 import re
22 from os import getenv, environ, getcwd
23 from os.path import isfile, join, expanduser
24 from typing import List, Union
25 from urllib.parse import urlparse, parse_qs
26
27 import dateutil.parser as date_parser
28 from .logger import get_logger
29
30 logger = get_logger()
31
32
33 class GzipStream(io.RawIOBase):
34 """Compress files on the fly.
35
36 GzipStream is a helper class around the gzip library. It helps to
37 compress already opened files (file-like objects) on the fly, so
38 there is no need to read everything into the memory and call the
39 `compress` function on it.
40 The GzipFile is opened on the instance itself so it needs to act
41 as a file-like object.
42
43 Args:
44 input: the source of the data to be compressed.
45 It can be a file-like object, bytes or string.
46 """
47
48 def __init__(self, source: Union[io.IOBase, bytes, str]):
49 self.buffer = b''
50
51 if isinstance(source, io.IOBase):
52 # The input is already a file-like object, use it as-is.
53 self.uncompressed = source
54 elif isinstance(source, str):
55 # Strings must be handled with StringIO.
56 self.uncompressed = io.StringIO(source)
57 else:
58 # Handle the rest as raw bytes.
59 self.uncompressed = io.BytesIO(source)
60
61 self.compressor = gzip.GzipFile(fileobj=self, mode='wb')
62
63 def read(self, size: int = -1) -> bytes:
64 """Compresses and returns the requested size of data.
65
66 Args:
67 size: how many bytes to return. -1 to read and compress the whole file
68 """
69 compressed = b''
70
71 if (size < 0) or (len(self.buffer) < size):
72 for raw in self.uncompressed:
73 # We need to encode text like streams (e.g. TextIOWrapper) to bytes.
74 if isinstance(raw, str):
75 raw = raw.encode()
76
77 self.compressor.write(raw)
78
79 # Stop compressing if we reached the max allowed size.
80 if 0 < size < len(self.buffer):
81 self.compressor.flush()
82 break
83 else:
84 self.compressor.close()
85
86 if size < 0:
87 # Return all data from the buffer.
88 compressed = self.buffer
89 self.buffer = b''
90 else:
91 # If we already have enough data in our buffer
92 # return the desired chunk of bytes
93 compressed = self.buffer[:size]
94 # then remove them from the buffer.
95 self.buffer = self.buffer[size:]
96
97 return compressed
98
99 def flush(self) -> None:
100 """Not implemented."""
101 # Since this "pipe" sits between 2 other stream (source/read -> target/write)
102 # it wouldn't be worth to implemet flushing.
103 pass
104
105 def write(self, compressed: bytes) -> None:
106 """Append the compressed data to the buffer
107
108 This happens when the target stream calls the `read` method and
109 that triggers the gzip "compressor".
110 """
111 self.buffer += compressed
112
113 def close(self) -> None:
114 """Closes the underlying file-like object."""
115 self.uncompressed.close()
116
117
118 def has_bad_first_or_last_char(val: str) -> bool:
119 """Returns true if a string starts with any of: {," ; or ends with any of: },".
120
121 Args:
122 val: The string to be tested.
123
124 Returns:
125 Whether or not the string starts or ends with bad characters.
126 """
127 return val is not None and (val.startswith('{') or val.startswith('"') or val.endswith('}') or val.endswith('"'))
128
129
130 def remove_null_values(dictionary: dict) -> dict:
131 """Create a new dictionary without keys mapped to null values.
132
133 Args:
134 dictionary: The dictionary potentially containing keys mapped to values of None.
135
136 Returns:
137 A dict with no keys mapped to None.
138 """
139 if isinstance(dictionary, dict):
140 return {k: v for (k, v) in dictionary.items() if v is not None}
141 return dictionary
142
143
144 def cleanup_values(dictionary: dict) -> dict:
145 """Create a new dictionary with boolean values converted to strings.
146
147 Ex. true -> 'true', false -> 'false'.
148 { 'key': true } -> { 'key': 'true' }
149
150 Args:
151 dictionary: The dictionary with keys mapped to booleans.
152
153 Returns:
154 The dictionary with certain keys mapped to s and not booleans.
155 """
156 if isinstance(dictionary, dict):
157 return {k: cleanup_value(v) for (k, v) in dictionary.items()}
158 return dictionary
159
160
161 def cleanup_value(value: any) -> any:
162 """Convert a boolean value to string."""
163 if isinstance(value, bool):
164 return 'true' if value else 'false'
165 return value
166
167
168 def strip_extra_slashes(value: str) -> str:
169 """Combine multiple trailing slashes to a single slash"""
170 if value.endswith('//'):
171 return value.rstrip('/') + '/'
172 return value
173
174
175 def datetime_to_string(val: datetime.datetime) -> str:
176 """Convert a datetime object to string.
177
178 If the supplied datetime does not specify a timezone,
179 it is assumed to be UTC.
180
181 Args:
182 val: The datetime object.
183
184 Returns:
185 datetime serialized to iso8601 format.
186 """
187 if isinstance(val, datetime.datetime):
188 if val.tzinfo is None:
189 return val.isoformat() + 'Z'
190 val = val.astimezone(datetime.timezone.utc)
191 return val.isoformat().replace('+00:00', 'Z')
192 return val
193
194
195 def string_to_datetime(string: str) -> datetime.datetime:
196 """De-serializes string to datetime.
197
198 Args:
199 string: string containing datetime in iso8601 format.
200
201 Returns:
202 the de-serialized string as a datetime object.
203 """
204 val = date_parser.parse(string)
205 if val.tzinfo is not None:
206 return val
207 return val.replace(tzinfo=datetime.timezone.utc)
208
209
210 def string_to_datetime_list(string_list: List[str]) -> List[datetime.datetime]:
211 """De-serializes each string in a list to a datetime.
212
213 Args:
214 string_list: list of strings containing datetime in iso8601 format.
215
216 Returns:
217 the de-serialized list of strings as a list of datetime objects.
218 """
219 if not isinstance(string_list, list):
220 raise ValueError(
221 "Invalid argument type: " + str(type(string_list)) + ". Argument string_list must be of type List[str]"
222 )
223 datetime_list = []
224 for string_val in string_list:
225 datetime_list.append(string_to_datetime(string_val))
226 return datetime_list
227
228
229 def datetime_to_string_list(datetime_list: List[datetime.datetime]) -> List[str]:
230 """Convert a list of datetime objects to a list of strings.
231
232 Args:
233 datetime_list: The list of datetime objects.
234
235 Returns:
236 list of datetimes serialized as strings in iso8601 format.
237 """
238 if not isinstance(datetime_list, list):
239 raise ValueError(
240 "Invalid argument type: "
241 + str(type(datetime_list))
242 + ". Argument datetime_list must be of type List[datetime.datetime]"
243 )
244 string_list = []
245 for datetime_val in datetime_list:
246 string_list.append(datetime_to_string(datetime_val))
247 return string_list
248
249
250 def date_to_string(val: datetime.date) -> str:
251 """Convert a date object to string.
252
253 Args:
254 val: The date object.
255
256 Returns:
257 date serialized to `YYYY-MM-DD` format.
258 """
259 if isinstance(val, datetime.date):
260 return str(val)
261 return val
262
263
264 def string_to_date(string: str) -> datetime.date:
265 """De-serializes string to date.
266
267 Args:
268 string: string containing date in 'YYYY-MM-DD' format.
269
270 Returns:
271 the de-serialized string as a date object.
272 """
273 return date_parser.parse(string).date()
274
275
276 def string_to_bool(string: str) -> bool:
277 """Converts 'string' to a bool value.
278 Args:
279 string: the value to convert. This should be some form of "true" or "false"
280 (e.g. "True", "TrUE", "False", "FaLsE", etc.)
281
282 Returns:
283 the boolean value
284 """
285 return string.strip().lower() == 'true'
286
287
288 def get_query_param(url_str: str, param: str) -> str:
289 """Return a query parameter value from url_str
290
291 Args:
292 url_str: the URL from which to extract the query
293 parameter value
294 param: the name of the query parameter whose value
295 should be returned
296
297 Returns:
298 the value of the `param` query parameter as a string
299
300 Raises:
301 ValueError: if errors are encountered parsing `url_str`
302 """
303 if not url_str:
304 return None
305 url = urlparse(url_str)
306 if not url.query:
307 return None
308 query = parse_qs(url.query, strict_parsing=True)
309 values = query.get(param)
310 return values[0] if values else None
311
312
313 def convert_model(val: any) -> dict:
314 """Convert a model object into an equivalent dict.
315
316 Arguments:
317 val: A dict or a model object
318
319 Returns:
320 A dict representation of the input object.
321 """
322 if isinstance(val, dict):
323 return val
324 if hasattr(val, "to_dict"):
325 return val.to_dict()
326 # Consider raising a ValueError here in the next major release
327 return val
328
329
330 def convert_list(val: Union[str, List[str]]) -> str:
331 """Convert a list of strings into comma-separated string.
332
333 Arguments:
334 val: A string or list of strings
335
336 Returns:
337 A comma-separated string of the items in the input list.
338 """
339 if isinstance(val, str):
340 return val
341 if isinstance(val, list) and all(isinstance(x, str) for x in val):
342 return ",".join(val)
343 # Consider raising a ValueError here in the next major release
344 return val
345
346
347 def read_external_sources(service_name: str) -> dict:
348 """Look for external configuration of a service.
349
350 Try to get config from external sources, with the following priority:
351 1. Credentials file(ibm-credentials.env)
352 2. Environment variables
353 3. VCAP Services(Cloud Foundry)
354
355 Args:
356 service_name: The service name
357
358 Returns:
359 A dictionary containing relevant configuration for the service if found.
360 """
361 logger.debug('Retrieving config properties for service \'%s\'', service_name)
362 config = {}
363
364 config = __read_from_credential_file(service_name)
365
366 if not config:
367 config = __read_from_env_variables(service_name)
368
369 if not config:
370 config = __read_from_vcap_services(service_name)
371 logger.debug('Retrieved %d properties', len(config))
372 return config
373
374
375 def __read_from_env_variables(service_name: str) -> dict:
376 """Return a config object based on environment variables for a service.
377
378 Args:
379 service_name: The name of the service to look for in env variables.
380
381 Returns:
382 A set of service configuration key-value pairs.
383 """
384 config = {}
385 for key, value in environ.items():
386 _parse_key_and_update_config(config, service_name, key, value)
387 return config
388
389
390 def __read_from_credential_file(service_name: str, *, separator: str = '=') -> dict:
391 """Return a config object based on credentials file for a service.
392
393 Args:
394 service_name: The name of the service to look for in env variables.
395
396 Keyword Args:
397 separator: The character to split on to de-serialize a line into a key-value pair.
398
399 Returns:
400 A set of service configuration key-value pairs.
401 """
402 default_credentials_file_name = 'ibm-credentials.env'
403
404 # 1. ${IBM_CREDENTIALS_FILE}
405 credential_file_path = getenv('IBM_CREDENTIALS_FILE')
406
407 # 2. <current-working-directory>/ibm-credentials.env
408 if credential_file_path is None:
409 file_path = join(getcwd(), default_credentials_file_name)
410 if isfile(file_path):
411 credential_file_path = file_path
412
413 # 3. <user-home-directory>/ibm-credentials.env
414 if credential_file_path is None:
415 file_path = join(expanduser('~'), default_credentials_file_name)
416 if isfile(file_path):
417 credential_file_path = file_path
418
419 config = {}
420 if credential_file_path is not None:
421 try:
422 with open(credential_file_path, 'r', encoding='utf-8') as fobj:
423 for line in fobj:
424 key_val = line.strip().split(separator, 1)
425 if len(key_val) == 2:
426 key = key_val[0]
427 value = key_val[1]
428 _parse_key_and_update_config(config, service_name, key, value)
429 except OSError:
430 # just absorb the exception and make sure we return an empty response
431 config = {}
432
433 return config
434
435
436 def _parse_key_and_update_config(config: dict, service_name: str, key: str, value: str) -> None:
437 service_name = service_name.replace(' ', '_').replace('-', '_').upper()
438 if key.startswith(service_name):
439 config[key[len(service_name) + 1 :]] = value
440
441
442 def __read_from_vcap_services(service_name: str) -> dict:
443 """Return a config object based on the vcap services environment variable.
444
445 Args:
446 service_name: The name of the service to look for in the vcap.
447
448 Returns:
449 A set of service configuration key-value pairs.
450 """
451 vcap_services = getenv('VCAP_SERVICES')
452 vcap_service_credentials = {}
453 if vcap_services:
454 services = json_import.loads(vcap_services)
455 for key in services.keys():
456 for i in range(len(services[key])):
457 if vcap_service_credentials and isinstance(vcap_service_credentials, dict):
458 break
459 if services[key][i].get('name') == service_name:
460 vcap_service_credentials = services[key][i].get('credentials', {})
461 if not vcap_service_credentials:
462 if service_name in services.keys():
463 service = services.get(service_name)
464 if service:
465 vcap_service_credentials = service[0].get('credentials', {})
466
467 if vcap_service_credentials and isinstance(vcap_service_credentials, dict):
468 new_vcap_creds = {}
469 # cf
470 if vcap_service_credentials.get('username') and vcap_service_credentials.get('password'):
471 new_vcap_creds['AUTH_TYPE'] = 'basic'
472 new_vcap_creds['USERNAME'] = vcap_service_credentials.get('username')
473 new_vcap_creds['PASSWORD'] = vcap_service_credentials.get('password')
474 vcap_service_credentials = new_vcap_creds
475 elif vcap_service_credentials.get('iam_apikey'):
476 new_vcap_creds['AUTH_TYPE'] = 'iam'
477 new_vcap_creds['APIKEY'] = vcap_service_credentials.get('iam_apikey')
478 vcap_service_credentials = new_vcap_creds
479 elif vcap_service_credentials.get('apikey'):
480 new_vcap_creds['AUTH_TYPE'] = 'iam'
481 new_vcap_creds['APIKEY'] = vcap_service_credentials.get('apikey')
482 vcap_service_credentials = new_vcap_creds
483 return vcap_service_credentials
484
485
486 # A regex that matches an "application/json" mimetype.
487 json_mimetype_pattern = re.compile('^application/json(\\s*;.*)?$')
488
489
490 def is_json_mimetype(mimetype: str) -> bool:
491 """Returns true if 'mimetype' is a JSON-like mimetype, false otherwise.
492
493 Args:
494 mimetype: The mimetype to check.
495
496 Returns:
497 true if mimetype is a JSON-line mimetype, false otherwise.
498 """
499 return mimetype is not None and json_mimetype_pattern.match(mimetype) is not None