| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
import random |
| 4 |
from typing import Union, AnyStr, IO, List, Optional, Dict |
| 5 |
from urllib.parse import urlparse |
| 6 |
|
| 7 |
from azure.storage.blob import BlobServiceClient |
| 8 |
|
| 9 |
from azure.core.tracing.decorator import distributed_trace |
| 10 |
from azure.core.tracing import SpanKind |
| 11 |
from azure.storage.queue import QueueServiceClient, TextBase64EncodePolicy |
| 12 |
|
| 13 |
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder |
| 14 |
from azure.kusto.data._telemetry import MonitoredActivity |
| 15 |
from azure.kusto.data.exceptions import KustoClosedError, KustoServiceError |
| 16 |
|
| 17 |
from ._ingest_telemetry import IngestTracingAttributes |
| 18 |
from ._resource_manager import _ResourceManager, _ResourceUri |
| 19 |
from .base_ingest_client import BaseIngestClient, IngestionResult, IngestionStatus |
| 20 |
from .descriptors import BlobDescriptor, FileDescriptor, StreamDescriptor |
| 21 |
from .exceptions import KustoInvalidEndpointError, KustoQueueError |
| 22 |
from azure.kusto.data.exceptions import KustoBlobError |
| 23 |
from .ingestion_blob_info import IngestionBlobInfo |
| 24 |
from .ingestion_properties import IngestionProperties |
| 25 |
|
| 26 |
|
| 27 |
class QueuedIngestClient(BaseIngestClient): |
| 28 |
""" |
| 29 |
Queued ingest client provides methods to allow queued ingestion into kusto (ADX). |
| 30 |
To learn more about the different types of ingestions and when to use each, visit: |
| 31 |
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods |
| 32 |
""" |
| 33 |
|
| 34 |
_INGEST_PREFIX = "ingest-" |
| 35 |
_SERVICE_CLIENT_TIMEOUT_SECONDS = 10 * 60 |
| 36 |
_MAX_RETRIES = 3 |
| 37 |
|
| 38 |
def __init__(self, kcsb: Union[str, KustoConnectionStringBuilder], auto_correct_endpoint: bool = True): |
| 39 |
"""Kusto Ingest Client constructor. |
| 40 |
:param kcsb: The connection string to initialize KustoClient. |
| 41 |
""" |
| 42 |
super().__init__() |
| 43 |
if not isinstance(kcsb, KustoConnectionStringBuilder): |
| 44 |
kcsb = KustoConnectionStringBuilder(kcsb) |
| 45 |
|
| 46 |
if auto_correct_endpoint: |
| 47 |
kcsb["Data Source"] = BaseIngestClient.get_ingestion_endpoint(kcsb.data_source) |
| 48 |
|
| 49 |
self._proxy_dict: Optional[Dict[str, str]] = None |
| 50 |
self._connection_datasource = kcsb.data_source |
| 51 |
self._resource_manager = _ResourceManager(KustoClient(kcsb)) |
| 52 |
self._endpoint_service_type = None |
| 53 |
self._suggested_endpoint_uri = None |
| 54 |
self.application_for_tracing = kcsb.client_details.application_for_tracing |
| 55 |
self.client_version_for_tracing = kcsb.client_details.version_for_tracing |
| 56 |
|
| 57 |
def close(self) -> None: |
| 58 |
self._resource_manager.close() |
| 59 |
super().close() |
| 60 |
|
| 61 |
def set_proxy(self, proxy_url: str): |
| 62 |
self._resource_manager.set_proxy(proxy_url) |
| 63 |
self._proxy_dict = {"http": proxy_url, "https": proxy_url} |
| 64 |
|
| 65 |
@distributed_trace(name_of_span="QueuedIngestClient.ingest_from_file", kind=SpanKind.CLIENT) |
| 66 |
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 67 |
"""Enqueue an ingest command from local files. |
| 68 |
To learn more about ingestion methods go to: |
| 69 |
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods |
| 70 |
:param file_descriptor: a FileDescriptor to be ingested. |
| 71 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 72 |
""" |
| 73 |
file_descriptor = FileDescriptor.get_instance(file_descriptor) |
| 74 |
IngestTracingAttributes.set_ingest_descriptor_attributes(file_descriptor, ingestion_properties) |
| 75 |
|
| 76 |
super().ingest_from_file(file_descriptor, ingestion_properties) |
| 77 |
|
| 78 |
containers = self._get_containers() |
| 79 |
|
| 80 |
file_descriptor, should_compress = BaseIngestClient._prepare_file(file_descriptor, ingestion_properties) |
| 81 |
with file_descriptor.open(should_compress) as stream: |
| 82 |
blob_descriptor = self.upload_blob( |
| 83 |
containers, |
| 84 |
file_descriptor, |
| 85 |
ingestion_properties.database, |
| 86 |
ingestion_properties.table, |
| 87 |
stream, |
| 88 |
self._proxy_dict, |
| 89 |
self._SERVICE_CLIENT_TIMEOUT_SECONDS, |
| 90 |
self._MAX_RETRIES, |
| 91 |
) |
| 92 |
return self.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties) |
| 93 |
|
| 94 |
@distributed_trace(name_of_span="QueuedIngestClient.ingest_from_stream", kind=SpanKind.CLIENT) |
| 95 |
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 96 |
"""Ingest from io streams. |
| 97 |
:param stream_descriptor: An object that contains a description of the stream to be ingested. |
| 98 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 99 |
""" |
| 100 |
stream_descriptor = StreamDescriptor.get_instance(stream_descriptor) |
| 101 |
IngestTracingAttributes.set_ingest_descriptor_attributes(stream_descriptor, ingestion_properties) |
| 102 |
|
| 103 |
super().ingest_from_stream(stream_descriptor, ingestion_properties) |
| 104 |
|
| 105 |
containers = self._get_containers() |
| 106 |
|
| 107 |
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties) |
| 108 |
blob_descriptor = self.upload_blob( |
| 109 |
containers, |
| 110 |
stream_descriptor, |
| 111 |
ingestion_properties.database, |
| 112 |
ingestion_properties.table, |
| 113 |
stream_descriptor.stream, |
| 114 |
self._proxy_dict, |
| 115 |
self._SERVICE_CLIENT_TIMEOUT_SECONDS, |
| 116 |
self._MAX_RETRIES, |
| 117 |
) |
| 118 |
return self.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties) |
| 119 |
|
| 120 |
@distributed_trace(name_of_span="QueuedIngestClient.ingest_from_blob", kind=SpanKind.CLIENT) |
| 121 |
def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties) -> IngestionResult: |
| 122 |
"""Enqueue an ingest command from azure blobs. |
| 123 |
To learn more about ingestion methods go to: |
| 124 |
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods |
| 125 |
:param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested. |
| 126 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 127 |
""" |
| 128 |
IngestTracingAttributes.set_ingest_descriptor_attributes(blob_descriptor, ingestion_properties) |
| 129 |
|
| 130 |
if self._is_closed: |
| 131 |
raise KustoClosedError() |
| 132 |
|
| 133 |
queues = self._resource_manager.get_ingestion_queues() |
| 134 |
|
| 135 |
authorization_context = self._resource_manager.get_authorization_context() |
| 136 |
ingestion_blob_info = IngestionBlobInfo( |
| 137 |
blob_descriptor, |
| 138 |
ingestion_properties=ingestion_properties, |
| 139 |
auth_context=authorization_context, |
| 140 |
application_for_tracing=self.application_for_tracing, |
| 141 |
client_version_for_tracing=self.client_version_for_tracing, |
| 142 |
) |
| 143 |
ingestion_blob_info_json = ingestion_blob_info.to_json() |
| 144 |
retries_left = min(self._MAX_RETRIES, len(queues)) |
| 145 |
for queue in queues: |
| 146 |
try: |
| 147 |
with QueueServiceClient(queue.account_uri, proxies=self._proxy_dict) as queue_service: |
| 148 |
with queue_service.get_queue_client(queue=queue.object_name, message_encode_policy=TextBase64EncodePolicy()) as queue_client: |
| 149 |
# trace enqueuing of blob for ingestion |
| 150 |
invoker = lambda: queue_client.send_message(content=ingestion_blob_info_json, timeout=self._SERVICE_CLIENT_TIMEOUT_SECONDS) |
| 151 |
enqueue_trace_attributes = IngestTracingAttributes.create_enqueue_request_attributes(queue_client.queue_name, blob_descriptor.source_id) |
| 152 |
MonitoredActivity.invoke(invoker, name_of_span="QueuedIngestClient.enqueue_request", tracing_attributes=enqueue_trace_attributes) |
| 153 |
|
| 154 |
self._resource_manager.report_resource_usage_result(queue.storage_account_name, True) |
| 155 |
return IngestionResult( |
| 156 |
IngestionStatus.QUEUED, ingestion_properties.database, ingestion_properties.table, blob_descriptor.source_id, blob_descriptor.path |
| 157 |
) |
| 158 |
except Exception as e: |
| 159 |
retries_left = retries_left - 1 |
| 160 |
# TODO: log the retry once we have a proper logging system |
| 161 |
self._resource_manager.report_resource_usage_result(queue.storage_account_name, False) |
| 162 |
if retries_left == 0: |
| 163 |
raise KustoQueueError() from e |
| 164 |
|
| 165 |
def _get_containers(self) -> List[_ResourceUri]: |
| 166 |
return self._resource_manager.get_containers() |
| 167 |
|
| 168 |
def upload_blob( |
| 169 |
self, |
| 170 |
containers: List[_ResourceUri], |
| 171 |
descriptor: Union[FileDescriptor, "StreamDescriptor"], |
| 172 |
database: str, |
| 173 |
table: str, |
| 174 |
stream: IO[AnyStr], |
| 175 |
proxy_dict: Optional[Dict[str, str]], |
| 176 |
timeout: int, |
| 177 |
max_retries: int, |
| 178 |
) -> "BlobDescriptor": |
| 179 |
""" |
| 180 |
Uploads and transforms FileDescriptor or StreamDescriptor into a BlobDescriptor instance |
| 181 |
:param List[_ResourceUri] containers: blob containers |
| 182 |
:param Union[FileDescriptor, "StreamDescriptor"] descriptor: |
| 183 |
:param string database: database to be ingested to |
| 184 |
:param string table: table to be ingested to |
| 185 |
:param IO[AnyStr] stream: stream to be ingested from |
| 186 |
:param Optional[Dict[str, str]] proxy_dict: proxy urls |
| 187 |
:param int timeout: Azure service call timeout in seconds |
| 188 |
:return new BlobDescriptor instance |
| 189 |
""" |
| 190 |
blob_name = "{db}__{table}__{guid}__{file}".format(db=database, table=table, guid=descriptor.source_id, file=descriptor.stream_name) |
| 191 |
|
| 192 |
retries_left = min(max_retries, len(containers)) |
| 193 |
for container in containers: |
| 194 |
try: |
| 195 |
blob_service = BlobServiceClient(container.account_uri, proxies=proxy_dict) |
| 196 |
blob_client = blob_service.get_blob_client(container=container.object_name, blob=blob_name) |
| 197 |
blob_client.upload_blob(data=stream, timeout=timeout) |
| 198 |
self._resource_manager.report_resource_usage_result(container.storage_account_name, True) |
| 199 |
return BlobDescriptor(blob_client.url, descriptor.size, descriptor.source_id) |
| 200 |
except Exception as e: |
| 201 |
retries_left = retries_left - 1 |
| 202 |
# TODO: log the retry once we have a proper logging system |
| 203 |
self._resource_manager.report_resource_usage_result(container.storage_account_name, False) |
| 204 |
if retries_left == 0: |
| 205 |
raise KustoBlobError(e) |