Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/ingest_client.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import random
4 from typing import Union, AnyStr, IO, List, Optional, Dict
5 from urllib.parse import urlparse
6
7 from azure.storage.blob import BlobServiceClient
8
9 from azure.core.tracing.decorator import distributed_trace
10 from azure.core.tracing import SpanKind
11 from azure.storage.queue import QueueServiceClient, TextBase64EncodePolicy
12
13 from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
14 from azure.kusto.data._telemetry import MonitoredActivity
15 from azure.kusto.data.exceptions import KustoClosedError, KustoServiceError
16
17 from ._ingest_telemetry import IngestTracingAttributes
18 from ._resource_manager import _ResourceManager, _ResourceUri
19 from .base_ingest_client import BaseIngestClient, IngestionResult, IngestionStatus
20 from .descriptors import BlobDescriptor, FileDescriptor, StreamDescriptor
21 from .exceptions import KustoInvalidEndpointError, KustoQueueError
22 from azure.kusto.data.exceptions import KustoBlobError
23 from .ingestion_blob_info import IngestionBlobInfo
24 from .ingestion_properties import IngestionProperties
25
26
27 class QueuedIngestClient(BaseIngestClient):
28 """
29 Queued ingest client provides methods to allow queued ingestion into kusto (ADX).
30 To learn more about the different types of ingestions and when to use each, visit:
31 https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
32 """
33
34 _INGEST_PREFIX = "ingest-"
35 _SERVICE_CLIENT_TIMEOUT_SECONDS = 10 * 60
36 _MAX_RETRIES = 3
37
38 def __init__(self, kcsb: Union[str, KustoConnectionStringBuilder], auto_correct_endpoint: bool = True):
39 """Kusto Ingest Client constructor.
40 :param kcsb: The connection string to initialize KustoClient.
41 """
42 super().__init__()
43 if not isinstance(kcsb, KustoConnectionStringBuilder):
44 kcsb = KustoConnectionStringBuilder(kcsb)
45
46 if auto_correct_endpoint:
47 kcsb["Data Source"] = BaseIngestClient.get_ingestion_endpoint(kcsb.data_source)
48
49 self._proxy_dict: Optional[Dict[str, str]] = None
50 self._connection_datasource = kcsb.data_source
51 self._resource_manager = _ResourceManager(KustoClient(kcsb))
52 self._endpoint_service_type = None
53 self._suggested_endpoint_uri = None
54 self.application_for_tracing = kcsb.client_details.application_for_tracing
55 self.client_version_for_tracing = kcsb.client_details.version_for_tracing
56
57 def close(self) -> None:
58 self._resource_manager.close()
59 super().close()
60
61 def set_proxy(self, proxy_url: str):
62 self._resource_manager.set_proxy(proxy_url)
63 self._proxy_dict = {"http": proxy_url, "https": proxy_url}
64
65 @distributed_trace(name_of_span="QueuedIngestClient.ingest_from_file", kind=SpanKind.CLIENT)
66 def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
67 """Enqueue an ingest command from local files.
68 To learn more about ingestion methods go to:
69 https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
70 :param file_descriptor: a FileDescriptor to be ingested.
71 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
72 """
73 file_descriptor = FileDescriptor.get_instance(file_descriptor)
74 IngestTracingAttributes.set_ingest_descriptor_attributes(file_descriptor, ingestion_properties)
75
76 super().ingest_from_file(file_descriptor, ingestion_properties)
77
78 containers = self._get_containers()
79
80 file_descriptor, should_compress = BaseIngestClient._prepare_file(file_descriptor, ingestion_properties)
81 with file_descriptor.open(should_compress) as stream:
82 blob_descriptor = self.upload_blob(
83 containers,
84 file_descriptor,
85 ingestion_properties.database,
86 ingestion_properties.table,
87 stream,
88 self._proxy_dict,
89 self._SERVICE_CLIENT_TIMEOUT_SECONDS,
90 self._MAX_RETRIES,
91 )
92 return self.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties)
93
94 @distributed_trace(name_of_span="QueuedIngestClient.ingest_from_stream", kind=SpanKind.CLIENT)
95 def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
96 """Ingest from io streams.
97 :param stream_descriptor: An object that contains a description of the stream to be ingested.
98 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
99 """
100 stream_descriptor = StreamDescriptor.get_instance(stream_descriptor)
101 IngestTracingAttributes.set_ingest_descriptor_attributes(stream_descriptor, ingestion_properties)
102
103 super().ingest_from_stream(stream_descriptor, ingestion_properties)
104
105 containers = self._get_containers()
106
107 stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
108 blob_descriptor = self.upload_blob(
109 containers,
110 stream_descriptor,
111 ingestion_properties.database,
112 ingestion_properties.table,
113 stream_descriptor.stream,
114 self._proxy_dict,
115 self._SERVICE_CLIENT_TIMEOUT_SECONDS,
116 self._MAX_RETRIES,
117 )
118 return self.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties)
119
120 @distributed_trace(name_of_span="QueuedIngestClient.ingest_from_blob", kind=SpanKind.CLIENT)
121 def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties) -> IngestionResult:
122 """Enqueue an ingest command from azure blobs.
123 To learn more about ingestion methods go to:
124 https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
125 :param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested.
126 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
127 """
128 IngestTracingAttributes.set_ingest_descriptor_attributes(blob_descriptor, ingestion_properties)
129
130 if self._is_closed:
131 raise KustoClosedError()
132
133 queues = self._resource_manager.get_ingestion_queues()
134
135 authorization_context = self._resource_manager.get_authorization_context()
136 ingestion_blob_info = IngestionBlobInfo(
137 blob_descriptor,
138 ingestion_properties=ingestion_properties,
139 auth_context=authorization_context,
140 application_for_tracing=self.application_for_tracing,
141 client_version_for_tracing=self.client_version_for_tracing,
142 )
143 ingestion_blob_info_json = ingestion_blob_info.to_json()
144 retries_left = min(self._MAX_RETRIES, len(queues))
145 for queue in queues:
146 try:
147 with QueueServiceClient(queue.account_uri, proxies=self._proxy_dict) as queue_service:
148 with queue_service.get_queue_client(queue=queue.object_name, message_encode_policy=TextBase64EncodePolicy()) as queue_client:
149 # trace enqueuing of blob for ingestion
150 invoker = lambda: queue_client.send_message(content=ingestion_blob_info_json, timeout=self._SERVICE_CLIENT_TIMEOUT_SECONDS)
151 enqueue_trace_attributes = IngestTracingAttributes.create_enqueue_request_attributes(queue_client.queue_name, blob_descriptor.source_id)
152 MonitoredActivity.invoke(invoker, name_of_span="QueuedIngestClient.enqueue_request", tracing_attributes=enqueue_trace_attributes)
153
154 self._resource_manager.report_resource_usage_result(queue.storage_account_name, True)
155 return IngestionResult(
156 IngestionStatus.QUEUED, ingestion_properties.database, ingestion_properties.table, blob_descriptor.source_id, blob_descriptor.path
157 )
158 except Exception as e:
159 retries_left = retries_left - 1
160 # TODO: log the retry once we have a proper logging system
161 self._resource_manager.report_resource_usage_result(queue.storage_account_name, False)
162 if retries_left == 0:
163 raise KustoQueueError() from e
164
165 def _get_containers(self) -> List[_ResourceUri]:
166 return self._resource_manager.get_containers()
167
168 def upload_blob(
169 self,
170 containers: List[_ResourceUri],
171 descriptor: Union[FileDescriptor, "StreamDescriptor"],
172 database: str,
173 table: str,
174 stream: IO[AnyStr],
175 proxy_dict: Optional[Dict[str, str]],
176 timeout: int,
177 max_retries: int,
178 ) -> "BlobDescriptor":
179 """
180 Uploads and transforms FileDescriptor or StreamDescriptor into a BlobDescriptor instance
181 :param List[_ResourceUri] containers: blob containers
182 :param Union[FileDescriptor, "StreamDescriptor"] descriptor:
183 :param string database: database to be ingested to
184 :param string table: table to be ingested to
185 :param IO[AnyStr] stream: stream to be ingested from
186 :param Optional[Dict[str, str]] proxy_dict: proxy urls
187 :param int timeout: Azure service call timeout in seconds
188 :return new BlobDescriptor instance
189 """
190 blob_name = "{db}__{table}__{guid}__{file}".format(db=database, table=table, guid=descriptor.source_id, file=descriptor.stream_name)
191
192 retries_left = min(max_retries, len(containers))
193 for container in containers:
194 try:
195 blob_service = BlobServiceClient(container.account_uri, proxies=proxy_dict)
196 blob_client = blob_service.get_blob_client(container=container.object_name, blob=blob_name)
197 blob_client.upload_blob(data=stream, timeout=timeout)
198 self._resource_manager.report_resource_usage_result(container.storage_account_name, True)
199 return BlobDescriptor(blob_client.url, descriptor.size, descriptor.source_id)
200 except Exception as e:
201 retries_left = retries_left - 1
202 # TODO: log the retry once we have a proper logging system
203 self._resource_manager.report_resource_usage_result(container.storage_account_name, False)
204 if retries_left == 0:
205 raise KustoBlobError(e)