| 1 |
import abc |
| 2 |
import io |
| 3 |
import json |
| 4 |
import uuid |
| 5 |
from datetime import timedelta |
| 6 |
from typing import Union, Optional, Any, NoReturn, ClassVar, TYPE_CHECKING |
| 7 |
from urllib.parse import urljoin |
| 8 |
|
| 9 |
from requests import Response, Session |
| 10 |
|
| 11 |
from azure.kusto.data._cloud_settings import CloudSettings |
| 12 |
from azure.kusto.data._token_providers import CloudInfoTokenProvider |
| 13 |
from .client_details import ClientDetails |
| 14 |
from .client_request_properties import ClientRequestProperties |
| 15 |
from .exceptions import KustoServiceError, KustoThrottlingError, KustoApiError |
| 16 |
from .kcsb import KustoConnectionStringBuilder |
| 17 |
from .kusto_trusted_endpoints import well_known_kusto_endpoints |
| 18 |
from .response import KustoResponseDataSet, KustoResponseDataSetV2, KustoResponseDataSetV1 |
| 19 |
from .security import _AadHelper |
| 20 |
|
| 21 |
if TYPE_CHECKING: |
| 22 |
import aiohttp |
| 23 |
|
| 24 |
|
| 25 |
class _KustoClientBase(abc.ABC): |
| 26 |
API_VERSION = "2024-12-12" |
| 27 |
|
| 28 |
_mgmt_default_timeout: ClassVar[timedelta] = timedelta(hours=1, seconds=30) |
| 29 |
_query_default_timeout: ClassVar[timedelta] = timedelta(minutes=4, seconds=30) |
| 30 |
_streaming_ingest_default_timeout: ClassVar[timedelta] = timedelta(minutes=10) |
| 31 |
_client_server_delta: ClassVar[timedelta] = timedelta(seconds=30) |
| 32 |
|
| 33 |
_aad_helper: _AadHelper |
| 34 |
client_details: ClientDetails |
| 35 |
_endpoint_validated = False |
| 36 |
_session: Union["aiohttp.ClientSession", "Session"] |
| 37 |
|
| 38 |
def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str], is_async): |
| 39 |
self._kcsb = kcsb |
| 40 |
self._proxy_url: Optional[str] = None |
| 41 |
if not isinstance(kcsb, KustoConnectionStringBuilder): |
| 42 |
self._kcsb = KustoConnectionStringBuilder(kcsb) |
| 43 |
self._kusto_cluster = self._kcsb.data_source |
| 44 |
|
| 45 |
# notice that in this context, federated actually just stands for aad auth, not aad federated auth (legacy code) |
| 46 |
self._aad_helper = _AadHelper(self._kcsb, is_async) if self._kcsb.aad_federated_security else None |
| 47 |
|
| 48 |
if not self._kusto_cluster.endswith("/"): |
| 49 |
self._kusto_cluster += "/" |
| 50 |
|
| 51 |
# Create a session object for connection pooling |
| 52 |
self._mgmt_endpoint = urljoin(self._kusto_cluster, "v1/rest/mgmt") |
| 53 |
self._query_endpoint = urljoin(self._kusto_cluster, "v2/rest/query") |
| 54 |
self._streaming_ingest_endpoint = urljoin(self._kusto_cluster, "v1/rest/ingest/") |
| 55 |
self._request_headers = { |
| 56 |
"Accept": "application/json", |
| 57 |
"Accept-Encoding": "gzip,deflate", |
| 58 |
"x-ms-version": self.API_VERSION, |
| 59 |
} |
| 60 |
|
| 61 |
self.client_details = self._kcsb.client_details |
| 62 |
self._is_closed: bool = False |
| 63 |
|
| 64 |
self.default_database = self._kcsb.initial_catalog |
| 65 |
|
| 66 |
def _get_database_or_default(self, database_name: Optional[str]) -> str: |
| 67 |
return database_name or self.default_database |
| 68 |
|
| 69 |
def close(self): |
| 70 |
self._is_closed = True |
| 71 |
|
| 72 |
def set_proxy(self, proxy_url: str): |
| 73 |
self._proxy_url = proxy_url |
| 74 |
if self._aad_helper: |
| 75 |
self._aad_helper.token_provider.set_proxy(proxy_url) |
| 76 |
if isinstance(self._session, Session): |
| 77 |
self._aad_helper.token_provider.set_session(self._session) |
| 78 |
|
| 79 |
def validate_endpoint(self): |
| 80 |
if not self._endpoint_validated and self._aad_helper is not None: |
| 81 |
if isinstance(self._aad_helper.token_provider, CloudInfoTokenProvider): |
| 82 |
endpoint = CloudSettings.get_cloud_info_for_cluster( |
| 83 |
self._kusto_cluster, |
| 84 |
self._aad_helper.token_provider._proxy_dict, |
| 85 |
self._session if isinstance(self._session, Session) else None, |
| 86 |
).login_endpoint |
| 87 |
well_known_kusto_endpoints.validate_trusted_endpoint( |
| 88 |
self._kusto_cluster, |
| 89 |
endpoint, |
| 90 |
) |
| 91 |
self._endpoint_validated = True |
| 92 |
|
| 93 |
@staticmethod |
| 94 |
def _kusto_parse_by_endpoint(endpoint: str, response_json: Any) -> KustoResponseDataSet: |
| 95 |
if endpoint.endswith("v2/rest/query"): |
| 96 |
return KustoResponseDataSetV2(response_json) |
| 97 |
return KustoResponseDataSetV1(response_json) |
| 98 |
|
| 99 |
@staticmethod |
| 100 |
def _handle_http_error( |
| 101 |
exception: Exception, |
| 102 |
endpoint: Optional[str], |
| 103 |
payload: Optional[io.IOBase], |
| 104 |
response: "Union[Response, aiohttp.ClientResponse]", |
| 105 |
status: int, |
| 106 |
response_json: Any, |
| 107 |
response_text: Optional[str], |
| 108 |
) -> NoReturn: |
| 109 |
if status == 404: |
| 110 |
if payload: |
| 111 |
raise KustoServiceError("The ingestion endpoint does not exist. Please enable streaming ingestion on your cluster.", response) from exception |
| 112 |
|
| 113 |
raise KustoServiceError(f"The requested endpoint '{endpoint}' does not exist.", response) from exception |
| 114 |
|
| 115 |
if status == 429: |
| 116 |
raise KustoThrottlingError("The request was throttled by the server.", response) from exception |
| 117 |
|
| 118 |
if status == 401: |
| 119 |
raise KustoServiceError(f"401. Missing adequate access rights.", response) from exception |
| 120 |
|
| 121 |
if payload: |
| 122 |
message = f"An error occurred while trying to ingest: Status: {status}, Reason: {response.reason}, Text: {response_text}." |
| 123 |
if response_json: |
| 124 |
raise KustoApiError(response_json, message, response) from exception |
| 125 |
|
| 126 |
raise KustoServiceError(message, response) from exception |
| 127 |
|
| 128 |
if response_json: |
| 129 |
raise KustoApiError(response_json, http_response=response) from exception |
| 130 |
|
| 131 |
if response_text: |
| 132 |
raise KustoServiceError(response_text, response) from exception |
| 133 |
|
| 134 |
raise KustoServiceError("Server error response contains no data.", response) from exception |
| 135 |
|
| 136 |
|
| 137 |
class ExecuteRequestParams: |
| 138 |
@staticmethod |
| 139 |
def _from_stream( |
| 140 |
stream: io.IOBase, |
| 141 |
properties: ClientRequestProperties, |
| 142 |
request_headers: Any, |
| 143 |
timeout: timedelta, |
| 144 |
mgmt_default_timeout: timedelta, |
| 145 |
client_server_delta: timedelta, |
| 146 |
client_details: ClientDetails, |
| 147 |
): |
| 148 |
# Before 3.0 it was KPC.execute_streaming_ingest, but was changed to align with the other SDKs |
| 149 |
client_request_id_prefix = "KPC.executeStreamingIngest;" |
| 150 |
request_headers = request_headers.copy() |
| 151 |
request_headers["Content-Encoding"] = "gzip" |
| 152 |
if properties: |
| 153 |
request_headers.update(json.loads(properties.to_json())["Options"]) |
| 154 |
|
| 155 |
return ExecuteRequestParams( |
| 156 |
stream, None, request_headers, client_request_id_prefix, properties, timeout, mgmt_default_timeout, client_server_delta, client_details |
| 157 |
) |
| 158 |
|
| 159 |
@staticmethod |
| 160 |
def _from_query( |
| 161 |
query: str, |
| 162 |
database: str, |
| 163 |
properties: ClientRequestProperties, |
| 164 |
request_headers: Any, |
| 165 |
timeout: timedelta, |
| 166 |
mgmt_default_timeout: timedelta, |
| 167 |
client_server_delta: timedelta, |
| 168 |
client_details: ClientDetails, |
| 169 |
): |
| 170 |
json_payload = {"db": database, "csl": query} |
| 171 |
if properties: |
| 172 |
json_payload["properties"] = properties.to_json() |
| 173 |
|
| 174 |
client_request_id_prefix = "KPC.execute;" |
| 175 |
request_headers = request_headers.copy() |
| 176 |
request_headers["Content-Type"] = "application/json; charset=utf-8" |
| 177 |
|
| 178 |
return ExecuteRequestParams( |
| 179 |
None, json_payload, request_headers, client_request_id_prefix, properties, timeout, mgmt_default_timeout, client_server_delta, client_details |
| 180 |
) |
| 181 |
|
| 182 |
@staticmethod |
| 183 |
def _from_blob_url( |
| 184 |
blob: str, |
| 185 |
properties: ClientRequestProperties, |
| 186 |
request_headers: Any, |
| 187 |
timeout: timedelta, |
| 188 |
mgmt_default_timeout: timedelta, |
| 189 |
client_server_delta: timedelta, |
| 190 |
client_details: ClientDetails, |
| 191 |
): |
| 192 |
json_payload = {"sourceUri": blob} |
| 193 |
client_request_id_prefix = "KPC.executeStreamingIngestFromBlob;" |
| 194 |
request_headers = request_headers.copy() |
| 195 |
request_headers["Content-Type"] = "application/json; charset=utf-8" |
| 196 |
if properties: |
| 197 |
request_headers.update(json.loads(properties.to_json())["Options"]) |
| 198 |
return ExecuteRequestParams( |
| 199 |
None, json_payload, request_headers, client_request_id_prefix, properties, timeout, mgmt_default_timeout, client_server_delta, client_details |
| 200 |
) |
| 201 |
|
| 202 |
def __init__( |
| 203 |
self, |
| 204 |
payload, |
| 205 |
json_payload, |
| 206 |
request_headers, |
| 207 |
client_request_id_prefix, |
| 208 |
properties: ClientRequestProperties, |
| 209 |
timeout: timedelta, |
| 210 |
mgmt_default_timeout: timedelta, |
| 211 |
client_server_delta: timedelta, |
| 212 |
client_details: ClientDetails, |
| 213 |
): |
| 214 |
special_headers = [ |
| 215 |
{ |
| 216 |
"name": "x-ms-client-request-id", |
| 217 |
"value": client_request_id_prefix + str(uuid.uuid4()), |
| 218 |
"property": lambda p: p.client_request_id, |
| 219 |
}, |
| 220 |
{ |
| 221 |
"name": "x-ms-client-version", |
| 222 |
"value": client_details.version_for_tracing, |
| 223 |
"property": lambda p: None, |
| 224 |
}, |
| 225 |
{ |
| 226 |
"name": "x-ms-app", |
| 227 |
"value": client_details.application_for_tracing, |
| 228 |
"property": lambda p: p.application, |
| 229 |
}, |
| 230 |
{ |
| 231 |
"name": "x-ms-user", |
| 232 |
"value": client_details.user_name_for_tracing, |
| 233 |
"property": lambda p: p.user, |
| 234 |
}, |
| 235 |
] |
| 236 |
|
| 237 |
for header in special_headers: |
| 238 |
value: str |
| 239 |
if properties and header["property"](properties) is not None: |
| 240 |
value = header["property"](properties) |
| 241 |
else: |
| 242 |
value = header["value"] |
| 243 |
|
| 244 |
if value is not None: |
| 245 |
# Replace any characters that aren't ascii with '?' |
| 246 |
value = value.encode("ascii", "replace").decode("ascii", "strict") |
| 247 |
request_headers[header["name"]] = value |
| 248 |
|
| 249 |
if properties is not None: |
| 250 |
if properties.get_option(ClientRequestProperties.no_request_timeout_option_name, False): |
| 251 |
timeout = mgmt_default_timeout |
| 252 |
else: |
| 253 |
timeout = properties.get_option(ClientRequestProperties.request_timeout_option_name, timeout) |
| 254 |
|
| 255 |
timeout = (timeout or mgmt_default_timeout) + client_server_delta |
| 256 |
|
| 257 |
self.json_payload = json_payload |
| 258 |
self.request_headers = request_headers |
| 259 |
self.timeout = timeout |
| 260 |
self.payload = payload |