Debian Codegraph @

azure-kusto-data/azure/kusto/data/client.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import socket
4 import sys
5 from datetime import timedelta
6 from typing import AnyStr, IO, List, Optional, TYPE_CHECKING, Tuple, Union
7
8 import requests
9 import requests.adapters
10 from requests import Response
11 from urllib3.connection import HTTPConnection
12
13 from azure.core.tracing.decorator import distributed_trace
14 from azure.core.tracing import SpanKind
15
16 from azure.kusto.data._telemetry import Span, MonitoredActivity
17 from azure.kusto.data.exceptions import KustoServiceError
18
19 from .client_base import ExecuteRequestParams, _KustoClientBase
20 from .client_request_properties import ClientRequestProperties
21 from .data_format import DataFormat
22 from .exceptions import KustoClosedError, KustoNetworkError
23
24 from .kcsb import KustoConnectionStringBuilder
25 from .response import KustoResponseDataSet, KustoStreamingResponseDataSet
26 from .streaming_response import JsonTokenReader, StreamingDataSetEnumerator
27
28 if TYPE_CHECKING:
29 pass
30
31
32 class HTTPAdapterWithSocketOptions(requests.adapters.HTTPAdapter):
33 def __init__(self, *args, **kwargs):
34 self.socket_options = kwargs.pop("socket_options", None)
35 super(HTTPAdapterWithSocketOptions, self).__init__(*args, **kwargs)
36
37 def __getstate__(self):
38 state = super(HTTPAdapterWithSocketOptions, self).__getstate__()
39 state["socket_options"] = self.socket_options
40 return state
41
42 def init_poolmanager(self, *args, **kwargs):
43 if self.socket_options is not None:
44 kwargs["socket_options"] = self.socket_options
45 super(HTTPAdapterWithSocketOptions, self).init_poolmanager(*args, **kwargs)
46
47
48 class KustoClient(_KustoClientBase):
49 """
50 Kusto client for Python.
51 The client is a wrapper around the Kusto REST API.
52 To read more about it, go to https://docs.microsoft.com/en-us/azure/kusto/api/rest/
53
54 The primary methods are:
55 `execute_query`: executes a KQL query against the Kusto service.
56 `execute_mgmt`: executes a KQL control command against the Kusto service.
57 """
58
59 _mgmt_default_timeout = timedelta(hours=1)
60 _query_default_timeout = timedelta(minutes=4)
61 _streaming_ingest_default_timeout = timedelta(minutes=10)
62 _client_server_delta = timedelta(seconds=30)
63
64 # The maximum amount of connections to be able to operate in parallel
65 _max_pool_size = 100
66
67 def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str]):
68 """
69 Kusto Client constructor.
70 :param kcsb: The connection string to initialize KustoClient.
71 :type kcsb: azure.kusto.data.KustoConnectionStringBuilder or str
72 """
73 super().__init__(kcsb, False)
74
75 # Create a session object for connection pooling
76 self._session = requests.Session()
77
78 adapter = HTTPAdapterWithSocketOptions(
79 socket_options=(HTTPConnection.default_socket_options or []) + self.compose_socket_options(), pool_maxsize=self._max_pool_size
80 )
81 self._session.mount("http://", adapter)
82 self._session.mount("https://", adapter)
83
84 def close(self):
85 if not self._is_closed:
86 self._session.close()
87 if self._aad_helper:
88 self._aad_helper.close()
89 super().close()
90
91 def __enter__(self):
92 return self
93
94 def __exit__(self, exc_type, exc_val, exc_tb):
95 self.close()
96
97 def set_proxy(self, proxy_url: str):
98 super().set_proxy(proxy_url)
99 self._session.proxies = {"http": proxy_url, "https": proxy_url}
100
101 def set_http_retries(self, max_retries: int):
102 """
103 Set the number of HTTP retries to attempt
104 """
105 adapter = HTTPAdapterWithSocketOptions(
106 socket_options=(HTTPConnection.default_socket_options or []) + self.compose_socket_options(),
107 pool_maxsize=self._max_pool_size,
108 max_retries=max_retries,
109 )
110 self._session.mount("http://", adapter)
111 self._session.mount("https://", adapter)
112
113 @staticmethod
114 def compose_socket_options() -> List[Tuple[int, int, int]]:
115 # Sends TCP Keep-Alive after MAX_IDLE_SECONDS seconds of idleness, once every INTERVAL_SECONDS seconds, and closes the connection after MAX_FAILED_KEEPALIVES failed pings (e.g. 20 => 1:00:30)
116 MAX_IDLE_SECONDS = 30
117 INTERVAL_SECONDS = 180 # Corresponds to Azure Load Balancer Service 4 minute timeout, with 1 minute of slack
118 MAX_FAILED_KEEPALIVES = 20
119
120 if (
121 sys.platform == "linux"
122 and hasattr(socket, "SOL_SOCKET")
123 and hasattr(socket, "SO_KEEPALIVE")
124 and hasattr(socket, "TCP_KEEPIDLE")
125 and hasattr(socket, "TCP_KEEPINTVL")
126 and hasattr(socket, "TCP_KEEPCNT")
127 ):
128 return [
129 (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
130 (socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, MAX_IDLE_SECONDS),
131 (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, INTERVAL_SECONDS),
132 (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, MAX_FAILED_KEEPALIVES),
133 ]
134 elif (
135 sys.platform == "win32"
136 and hasattr(socket, "SOL_SOCKET")
137 and hasattr(socket, "SO_KEEPALIVE")
138 and hasattr(socket, "TCP_KEEPIDLE")
139 and hasattr(socket, "TCP_KEEPCNT")
140 ):
141 return [
142 (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
143 (socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, MAX_IDLE_SECONDS),
144 (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, MAX_FAILED_KEEPALIVES),
145 ]
146 elif sys.platform == "darwin" and hasattr(socket, "SOL_SOCKET") and hasattr(socket, "SO_KEEPALIVE") and hasattr(socket, "IPPROTO_TCP"):
147 TCP_KEEPALIVE = 0x10
148 return [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), (socket.IPPROTO_TCP, TCP_KEEPALIVE, INTERVAL_SECONDS)]
149 else:
150 return []
151
152 def execute(self, database: Optional[str], query: str, properties: Optional[ClientRequestProperties] = None) -> KustoResponseDataSet:
153 """
154 Executes a query or management command.
155 :param Optional[str] database: Database against query will be executed. If not provided, will default to the "Initial Catalog" value in the connection string
156 :param str query: Query to be executed.
157 :param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
158 :return: Kusto response data set.
159 :rtype: azure.kusto.data.response.KustoResponseDataSet
160 """
161 query = query.strip()
162 if query.startswith("."):
163 return self.execute_mgmt(database, query, properties)
164 return self.execute_query(database, query, properties)
165
166 @distributed_trace(name_of_span="KustoClient.query_cmd", kind=SpanKind.CLIENT)
167 def execute_query(self, database: Optional[str], query: str, properties: Optional[ClientRequestProperties] = None) -> KustoResponseDataSet:
168 """
169 Execute a KQL query.
170 To learn more about KQL go to https://docs.microsoft.com/en-us/azure/kusto/query/
171 :param Optional[str] database: Database against query will be executed. If not provided, will default to the "Initial Catalog" value in the connection string
172 :param str query: Query to be executed.
173 :param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
174 :return: Kusto response data set.
175 :rtype: azure.kusto.data.response.KustoResponseDataSet
176 """
177 database = self._get_database_or_default(database)
178 Span.set_query_attributes(self._kusto_cluster, database, properties)
179 request = ExecuteRequestParams._from_query(
180 query,
181 database,
182 properties,
183 self._request_headers,
184 self._query_default_timeout,
185 self._mgmt_default_timeout,
186 self._client_server_delta,
187 self.client_details,
188 )
189 return self._execute(self._query_endpoint, request, properties)
190
191 @distributed_trace(name_of_span="KustoClient.control_cmd", kind=SpanKind.CLIENT)
192 def execute_mgmt(self, database: Optional[str], query: str, properties: Optional[ClientRequestProperties] = None) -> KustoResponseDataSet:
193 """
194 Execute a KQL control command.
195 To learn more about KQL control commands go to https://docs.microsoft.com/en-us/azure/kusto/management/
196 :param Optional[str] database: Database against query will be executed. If not provided, will default to the "Initial Catalog" value in the connection string
197 :param str query: Query to be executed.
198 :param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
199 :return: Kusto response data set.
200 :rtype: azure.kusto.data.response.KustoResponseDataSet
201 """
202 database = self._get_database_or_default(database)
203 Span.set_query_attributes(self._kusto_cluster, database, properties)
204 request = ExecuteRequestParams._from_query(
205 query,
206 database,
207 properties,
208 self._request_headers,
209 self._mgmt_default_timeout,
210 self._mgmt_default_timeout,
211 self._client_server_delta,
212 self.client_details,
213 )
214 return self._execute(self._mgmt_endpoint, request, properties)
215
216 @distributed_trace(name_of_span="KustoClient.streaming_ingest", kind=SpanKind.CLIENT)
217 def execute_streaming_ingest(
218 self,
219 database: Optional[str],
220 table: str,
221 stream: Optional[IO[AnyStr]],
222 blob_url: Optional[str],
223 stream_format: Union[DataFormat, str],
224 properties: Optional[ClientRequestProperties] = None,
225 mapping_name: str = None,
226 ):
227 """
228 Execute streaming ingest against this client
229 If the Kusto service is not configured to allow streaming ingestion, this may raise an error
230 To learn more about streaming ingestion go to:
231 https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-streaming
232 :param Optional[str] database: Target database. If not provided, will default to the "Initial Catalog" value in the connection string
233 :param str table: Target table.
234 :param Optional[IO[AnyStr]] stream: a stream object or which contains the data to ingest.
235 :param Optional[str] blob_url: An url to a blob which contains the data to ingest. Provide either this or stream.
236 :param DataFormat stream_format: Format of the data in the stream.
237 :param ClientRequestProperties properties: additional request properties.
238 :param str mapping_name: Pre-defined mapping of the table. Required when stream_format is json/avro.
239 """
240 database = self._get_database_or_default(database)
241
242 stream_format = stream_format.kusto_value if isinstance(stream_format, DataFormat) else DataFormat[stream_format.upper()].kusto_value
243 endpoint = self._streaming_ingest_endpoint + database + "/" + table + "?streamFormat=" + stream_format
244 if mapping_name is not None:
245 endpoint = endpoint + "&mappingName=" + mapping_name
246 if blob_url:
247 endpoint += "&sourceKind=uri"
248 request = ExecuteRequestParams._from_blob_url(
249 blob_url,
250 properties,
251 self._request_headers,
252 self._streaming_ingest_default_timeout,
253 self._mgmt_default_timeout,
254 self._client_server_delta,
255 self.client_details,
256 )
257 elif stream:
258 request = ExecuteRequestParams._from_stream(
259 stream,
260 properties,
261 self._request_headers,
262 self._streaming_ingest_default_timeout,
263 self._mgmt_default_timeout,
264 self._client_server_delta,
265 self.client_details,
266 )
267 else:
268 raise Exception("execute_streaming_ingest is expecting either a stream or blob url")
269
270 Span.set_streaming_ingest_attributes(self._kusto_cluster, database, table, properties)
271 self._execute(endpoint, request, properties)
272
273 def _execute_streaming_query_parsed(
274 self,
275 database: Optional[str],
276 query: str,
277 timeout: timedelta = _KustoClientBase._query_default_timeout,
278 properties: Optional[ClientRequestProperties] = None,
279 ) -> StreamingDataSetEnumerator:
280 request = ExecuteRequestParams._from_query(
281 query, database, properties, self._request_headers, timeout, self._mgmt_default_timeout, self._client_server_delta, self.client_details
282 )
283 response = self._execute(self._query_endpoint, request, properties, stream_response=True)
284 response.raw.decode_content = True
285 return StreamingDataSetEnumerator(JsonTokenReader(response.raw))
286
287 @distributed_trace(name_of_span="KustoClient.streaming_query", kind=SpanKind.CLIENT)
288 def execute_streaming_query(
289 self,
290 database: Optional[str],
291 query: str,
292 timeout: timedelta = _KustoClientBase._query_default_timeout,
293 properties: Optional[ClientRequestProperties] = None,
294 ) -> KustoStreamingResponseDataSet:
295 """
296 Execute a KQL query without reading it all to memory.
297 The resulting KustoStreamingResponseDataSet will stream one table at a time, and the rows can be retrieved sequentially.
298
299 :param Optional[str] database: Database against query will be executed. If not provided, will default to the "Initial Catalog" value in the connection string
300 :param str query: Query to be executed.
301 :param timedelta timeout: timeout for the query to be executed
302 :param azure.kusto.data.ClientRequestProperties properties: Optional additional properties.
303 :return KustoStreamingResponseDataSet:
304 """
305 Span.set_query_attributes(self._kusto_cluster, database, properties)
306
307 return KustoStreamingResponseDataSet(self._execute_streaming_query_parsed(database, query, timeout, properties))
308
309 def _execute(
310 self,
311 endpoint: str,
312 request: ExecuteRequestParams,
313 properties: Optional[ClientRequestProperties] = None,
314 stream_response: bool = False,
315 ) -> Union[KustoResponseDataSet, Response]:
316 """Executes given query against this client"""
317 if self._is_closed:
318 raise KustoClosedError()
319 self.validate_endpoint()
320
321 request_headers = request.request_headers
322 if self._aad_helper:
323 request_headers["Authorization"] = self._aad_helper.acquire_authorization_header()
324
325 # trace http post call for response
326 invoker = lambda: self._session.post(
327 endpoint,
328 headers=request_headers,
329 json=request.json_payload,
330 data=request.payload,
331 timeout=request.timeout.seconds,
332 stream=stream_response,
333 allow_redirects=False,
334 )
335
336 try:
337 response = MonitoredActivity.invoke(
338 invoker, name_of_span="KustoClient.http_post", tracing_attributes=Span.create_http_attributes("POST", endpoint, request_headers)
339 )
340 except Exception as e:
341 raise KustoNetworkError(endpoint, None if properties is None else properties.client_request_id) from e
342
343 if stream_response:
344 try:
345 response.raise_for_status()
346 if 300 <= response.status_code < 400:
347 raise Exception("Unexpected redirection, got status code: " + str(response.status))
348 return response
349 except Exception as e:
350 raise self._handle_http_error(e, self._query_endpoint, None, response, response.status_code, response.json(), response.text)
351
352 response_json = None
353 try:
354 if 300 <= response.status_code < 400:
355 raise Exception("Unexpected redirection, got status code: " + str(response.status))
356 if response.text:
357 response_json = response.json()
358 else:
359 raise KustoServiceError("The content of the response contains no data.", response)
360 response.raise_for_status()
361 except Exception as e:
362 raise self._handle_http_error(e, endpoint, request.payload, response, response.status_code, response_json, response.text)
363 # trace response processing
364 return MonitoredActivity.invoke(lambda: self._kusto_parse_by_endpoint(endpoint, response_json), name_of_span="KustoClient.processing_response")