| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
from typing import Union, AnyStr, Optional |
| 4 |
from typing import IO |
| 5 |
|
| 6 |
from azure.core.tracing.decorator import distributed_trace |
| 7 |
from azure.core.tracing import SpanKind |
| 8 |
|
| 9 |
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties |
| 10 |
|
| 11 |
from ._ingest_telemetry import IngestTracingAttributes |
| 12 |
from .base_ingest_client import BaseIngestClient, IngestionResult, IngestionStatus |
| 13 |
from .descriptors import FileDescriptor, StreamDescriptor, BlobDescriptor |
| 14 |
from .ingestion_properties import IngestionProperties |
| 15 |
|
| 16 |
|
| 17 |
class KustoStreamingIngestClient(BaseIngestClient): |
| 18 |
"""Kusto streaming ingest client for Python. |
| 19 |
KustoStreamingIngestClient works with both 2.x and 3.x flavors of Python. |
| 20 |
All primitive types are supported. |
| 21 |
Tests are run using pytest. |
| 22 |
""" |
| 23 |
|
| 24 |
def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str], auto_correct_endpoint: bool = True): |
| 25 |
"""Kusto Streaming Ingest Client constructor. |
| 26 |
:param KustoConnectionStringBuilder kcsb: The connection string to initialize KustoClient. |
| 27 |
""" |
| 28 |
super().__init__() |
| 29 |
|
| 30 |
if isinstance(kcsb, str): |
| 31 |
kcsb = KustoConnectionStringBuilder(kcsb) |
| 32 |
|
| 33 |
if auto_correct_endpoint: |
| 34 |
kcsb["Data Source"] = BaseIngestClient.get_query_endpoint(kcsb.data_source) |
| 35 |
self._kusto_client = KustoClient(kcsb) |
| 36 |
|
| 37 |
def close(self): |
| 38 |
if not self._is_closed: |
| 39 |
self._kusto_client.close() |
| 40 |
super().close() |
| 41 |
|
| 42 |
def set_proxy(self, proxy_url: str): |
| 43 |
self._kusto_client.set_proxy(proxy_url) |
| 44 |
|
| 45 |
@distributed_trace(kind=SpanKind.CLIENT) |
| 46 |
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 47 |
"""Ingest from local files. |
| 48 |
:param file_descriptor: a FileDescriptor to be ingested. |
| 49 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 50 |
""" |
| 51 |
file_descriptor = FileDescriptor.get_instance(file_descriptor) |
| 52 |
IngestTracingAttributes.set_ingest_descriptor_attributes(file_descriptor, ingestion_properties) |
| 53 |
|
| 54 |
super().ingest_from_file(file_descriptor, ingestion_properties) |
| 55 |
|
| 56 |
stream_descriptor = StreamDescriptor.from_file_descriptor(file_descriptor) |
| 57 |
|
| 58 |
with stream_descriptor.stream: |
| 59 |
return self.ingest_from_stream(stream_descriptor, ingestion_properties) |
| 60 |
|
| 61 |
@distributed_trace(kind=SpanKind.CLIENT) |
| 62 |
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 63 |
"""Ingest from io streams. |
| 64 |
:param azure.kusto.ingest.StreamDescriptor stream_descriptor: An object that contains a description of the stream to |
| 65 |
be ingested. |
| 66 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 67 |
""" |
| 68 |
stream_descriptor = StreamDescriptor.get_instance(stream_descriptor) |
| 69 |
IngestTracingAttributes.set_ingest_descriptor_attributes(stream_descriptor, ingestion_properties) |
| 70 |
|
| 71 |
super().ingest_from_stream(stream_descriptor, ingestion_properties) |
| 72 |
|
| 73 |
return self._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, None) |
| 74 |
|
| 75 |
def _ingest_from_stream_with_client_request_id( |
| 76 |
self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties, client_request_id: Optional[str] |
| 77 |
) -> IngestionResult: |
| 78 |
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties) |
| 79 |
additional_properties = None |
| 80 |
if client_request_id: |
| 81 |
additional_properties = ClientRequestProperties() |
| 82 |
additional_properties.client_request_id = client_request_id |
| 83 |
|
| 84 |
self._kusto_client.execute_streaming_ingest( |
| 85 |
ingestion_properties.database, |
| 86 |
ingestion_properties.table, |
| 87 |
stream_descriptor.stream, |
| 88 |
None, |
| 89 |
ingestion_properties.format.name, |
| 90 |
additional_properties, |
| 91 |
mapping_name=ingestion_properties.ingestion_mapping_reference, |
| 92 |
) |
| 93 |
|
| 94 |
return IngestionResult(IngestionStatus.SUCCESS, ingestion_properties.database, ingestion_properties.table, stream_descriptor.source_id) |
| 95 |
|
| 96 |
def ingest_from_blob( |
| 97 |
self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties, client_request_id: Optional[str] = None |
| 98 |
) -> IngestionResult: |
| 99 |
IngestTracingAttributes.set_ingest_descriptor_attributes(blob_descriptor, ingestion_properties) |
| 100 |
additional_properties = None |
| 101 |
if client_request_id: |
| 102 |
additional_properties = ClientRequestProperties() |
| 103 |
additional_properties.client_request_id = client_request_id |
| 104 |
|
| 105 |
self._kusto_client.execute_streaming_ingest( |
| 106 |
ingestion_properties.database, |
| 107 |
ingestion_properties.table, |
| 108 |
None, |
| 109 |
blob_descriptor.path, |
| 110 |
ingestion_properties.format.name, |
| 111 |
additional_properties, |
| 112 |
mapping_name=ingestion_properties.ingestion_mapping_reference, |
| 113 |
) |
| 114 |
return IngestionResult(IngestionStatus.SUCCESS, ingestion_properties.database, ingestion_properties.table, blob_descriptor.source_id) |