Debian Codegraph @

azure-kusto-data/azure/kusto/data/client_base.py

raw
1 import abc
2 import io
3 import json
4 import uuid
5 from datetime import timedelta
6 from typing import Union, Optional, Any, NoReturn, ClassVar, TYPE_CHECKING
7 from urllib.parse import urljoin
8
9 from requests import Response, Session
10
11 from azure.kusto.data._cloud_settings import CloudSettings
12 from azure.kusto.data._token_providers import CloudInfoTokenProvider
13 from .client_details import ClientDetails
14 from .client_request_properties import ClientRequestProperties
15 from .exceptions import KustoServiceError, KustoThrottlingError, KustoApiError
16 from .kcsb import KustoConnectionStringBuilder
17 from .kusto_trusted_endpoints import well_known_kusto_endpoints
18 from .response import KustoResponseDataSet, KustoResponseDataSetV2, KustoResponseDataSetV1
19 from .security import _AadHelper
20
21 if TYPE_CHECKING:
22 import aiohttp
23
24
25 class _KustoClientBase(abc.ABC):
26 API_VERSION = "2024-12-12"
27
28 _mgmt_default_timeout: ClassVar[timedelta] = timedelta(hours=1, seconds=30)
29 _query_default_timeout: ClassVar[timedelta] = timedelta(minutes=4, seconds=30)
30 _streaming_ingest_default_timeout: ClassVar[timedelta] = timedelta(minutes=10)
31 _client_server_delta: ClassVar[timedelta] = timedelta(seconds=30)
32
33 _aad_helper: _AadHelper
34 client_details: ClientDetails
35 _endpoint_validated = False
36 _session: Union["aiohttp.ClientSession", "Session"]
37
38 def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str], is_async):
39 self._kcsb = kcsb
40 self._proxy_url: Optional[str] = None
41 if not isinstance(kcsb, KustoConnectionStringBuilder):
42 self._kcsb = KustoConnectionStringBuilder(kcsb)
43 self._kusto_cluster = self._kcsb.data_source
44
45 # notice that in this context, federated actually just stands for aad auth, not aad federated auth (legacy code)
46 self._aad_helper = _AadHelper(self._kcsb, is_async) if self._kcsb.aad_federated_security else None
47
48 if not self._kusto_cluster.endswith("/"):
49 self._kusto_cluster += "/"
50
51 # Create a session object for connection pooling
52 self._mgmt_endpoint = urljoin(self._kusto_cluster, "v1/rest/mgmt")
53 self._query_endpoint = urljoin(self._kusto_cluster, "v2/rest/query")
54 self._streaming_ingest_endpoint = urljoin(self._kusto_cluster, "v1/rest/ingest/")
55 self._request_headers = {
56 "Accept": "application/json",
57 "Accept-Encoding": "gzip,deflate",
58 "x-ms-version": self.API_VERSION,
59 }
60
61 self.client_details = self._kcsb.client_details
62 self._is_closed: bool = False
63
64 self.default_database = self._kcsb.initial_catalog
65
66 def _get_database_or_default(self, database_name: Optional[str]) -> str:
67 return database_name or self.default_database
68
69 def close(self):
70 self._is_closed = True
71
72 def set_proxy(self, proxy_url: str):
73 self._proxy_url = proxy_url
74 if self._aad_helper:
75 self._aad_helper.token_provider.set_proxy(proxy_url)
76 if isinstance(self._session, Session):
77 self._aad_helper.token_provider.set_session(self._session)
78
79 def validate_endpoint(self):
80 if not self._endpoint_validated and self._aad_helper is not None:
81 if isinstance(self._aad_helper.token_provider, CloudInfoTokenProvider):
82 endpoint = CloudSettings.get_cloud_info_for_cluster(
83 self._kusto_cluster,
84 self._aad_helper.token_provider._proxy_dict,
85 self._session if isinstance(self._session, Session) else None,
86 ).login_endpoint
87 well_known_kusto_endpoints.validate_trusted_endpoint(
88 self._kusto_cluster,
89 endpoint,
90 )
91 self._endpoint_validated = True
92
93 @staticmethod
94 def _kusto_parse_by_endpoint(endpoint: str, response_json: Any) -> KustoResponseDataSet:
95 if endpoint.endswith("v2/rest/query"):
96 return KustoResponseDataSetV2(response_json)
97 return KustoResponseDataSetV1(response_json)
98
99 @staticmethod
100 def _handle_http_error(
101 exception: Exception,
102 endpoint: Optional[str],
103 payload: Optional[io.IOBase],
104 response: "Union[Response, aiohttp.ClientResponse]",
105 status: int,
106 response_json: Any,
107 response_text: Optional[str],
108 ) -> NoReturn:
109 if status == 404:
110 if payload:
111 raise KustoServiceError("The ingestion endpoint does not exist. Please enable streaming ingestion on your cluster.", response) from exception
112
113 raise KustoServiceError(f"The requested endpoint '{endpoint}' does not exist.", response) from exception
114
115 if status == 429:
116 raise KustoThrottlingError("The request was throttled by the server.", response) from exception
117
118 if status == 401:
119 raise KustoServiceError(f"401. Missing adequate access rights.", response) from exception
120
121 if payload:
122 message = f"An error occurred while trying to ingest: Status: {status}, Reason: {response.reason}, Text: {response_text}."
123 if response_json:
124 raise KustoApiError(response_json, message, response) from exception
125
126 raise KustoServiceError(message, response) from exception
127
128 if response_json:
129 raise KustoApiError(response_json, http_response=response) from exception
130
131 if response_text:
132 raise KustoServiceError(response_text, response) from exception
133
134 raise KustoServiceError("Server error response contains no data.", response) from exception
135
136
137 class ExecuteRequestParams:
138 @staticmethod
139 def _from_stream(
140 stream: io.IOBase,
141 properties: ClientRequestProperties,
142 request_headers: Any,
143 timeout: timedelta,
144 mgmt_default_timeout: timedelta,
145 client_server_delta: timedelta,
146 client_details: ClientDetails,
147 ):
148 # Before 3.0 it was KPC.execute_streaming_ingest, but was changed to align with the other SDKs
149 client_request_id_prefix = "KPC.executeStreamingIngest;"
150 request_headers = request_headers.copy()
151 request_headers["Content-Encoding"] = "gzip"
152 if properties:
153 request_headers.update(json.loads(properties.to_json())["Options"])
154
155 return ExecuteRequestParams(
156 stream, None, request_headers, client_request_id_prefix, properties, timeout, mgmt_default_timeout, client_server_delta, client_details
157 )
158
159 @staticmethod
160 def _from_query(
161 query: str,
162 database: str,
163 properties: ClientRequestProperties,
164 request_headers: Any,
165 timeout: timedelta,
166 mgmt_default_timeout: timedelta,
167 client_server_delta: timedelta,
168 client_details: ClientDetails,
169 ):
170 json_payload = {"db": database, "csl": query}
171 if properties:
172 json_payload["properties"] = properties.to_json()
173
174 client_request_id_prefix = "KPC.execute;"
175 request_headers = request_headers.copy()
176 request_headers["Content-Type"] = "application/json; charset=utf-8"
177
178 return ExecuteRequestParams(
179 None, json_payload, request_headers, client_request_id_prefix, properties, timeout, mgmt_default_timeout, client_server_delta, client_details
180 )
181
182 @staticmethod
183 def _from_blob_url(
184 blob: str,
185 properties: ClientRequestProperties,
186 request_headers: Any,
187 timeout: timedelta,
188 mgmt_default_timeout: timedelta,
189 client_server_delta: timedelta,
190 client_details: ClientDetails,
191 ):
192 json_payload = {"sourceUri": blob}
193 client_request_id_prefix = "KPC.executeStreamingIngestFromBlob;"
194 request_headers = request_headers.copy()
195 request_headers["Content-Type"] = "application/json; charset=utf-8"
196 if properties:
197 request_headers.update(json.loads(properties.to_json())["Options"])
198 return ExecuteRequestParams(
199 None, json_payload, request_headers, client_request_id_prefix, properties, timeout, mgmt_default_timeout, client_server_delta, client_details
200 )
201
202 def __init__(
203 self,
204 payload,
205 json_payload,
206 request_headers,
207 client_request_id_prefix,
208 properties: ClientRequestProperties,
209 timeout: timedelta,
210 mgmt_default_timeout: timedelta,
211 client_server_delta: timedelta,
212 client_details: ClientDetails,
213 ):
214 special_headers = [
215 {
216 "name": "x-ms-client-request-id",
217 "value": client_request_id_prefix + str(uuid.uuid4()),
218 "property": lambda p: p.client_request_id,
219 },
220 {
221 "name": "x-ms-client-version",
222 "value": client_details.version_for_tracing,
223 "property": lambda p: None,
224 },
225 {
226 "name": "x-ms-app",
227 "value": client_details.application_for_tracing,
228 "property": lambda p: p.application,
229 },
230 {
231 "name": "x-ms-user",
232 "value": client_details.user_name_for_tracing,
233 "property": lambda p: p.user,
234 },
235 ]
236
237 for header in special_headers:
238 value: str
239 if properties and header["property"](properties) is not None:
240 value = header["property"](properties)
241 else:
242 value = header["value"]
243
244 if value is not None:
245 # Replace any characters that aren't ascii with '?'
246 value = value.encode("ascii", "replace").decode("ascii", "strict")
247 request_headers[header["name"]] = value
248
249 if properties is not None:
250 if properties.get_option(ClientRequestProperties.no_request_timeout_option_name, False):
251 timeout = mgmt_default_timeout
252 else:
253 timeout = properties.get_option(ClientRequestProperties.request_timeout_option_name, timeout)
254
255 timeout = (timeout or mgmt_default_timeout) + client_server_delta
256
257 self.json_payload = json_payload
258 self.request_headers = request_headers
259 self.timeout = timeout
260 self.payload = payload