Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/streaming_ingest_client.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 from typing import Union, AnyStr, Optional
4 from typing import IO
5
6 from azure.core.tracing.decorator import distributed_trace
7 from azure.core.tracing import SpanKind
8
9 from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
10
11 from ._ingest_telemetry import IngestTracingAttributes
12 from .base_ingest_client import BaseIngestClient, IngestionResult, IngestionStatus
13 from .descriptors import FileDescriptor, StreamDescriptor, BlobDescriptor
14 from .ingestion_properties import IngestionProperties
15
16
17 class KustoStreamingIngestClient(BaseIngestClient):
18 """Kusto streaming ingest client for Python.
19 KustoStreamingIngestClient works with both 2.x and 3.x flavors of Python.
20 All primitive types are supported.
21 Tests are run using pytest.
22 """
23
24 def __init__(self, kcsb: Union[KustoConnectionStringBuilder, str], auto_correct_endpoint: bool = True):
25 """Kusto Streaming Ingest Client constructor.
26 :param KustoConnectionStringBuilder kcsb: The connection string to initialize KustoClient.
27 """
28 super().__init__()
29
30 if isinstance(kcsb, str):
31 kcsb = KustoConnectionStringBuilder(kcsb)
32
33 if auto_correct_endpoint:
34 kcsb["Data Source"] = BaseIngestClient.get_query_endpoint(kcsb.data_source)
35 self._kusto_client = KustoClient(kcsb)
36
37 def close(self):
38 if not self._is_closed:
39 self._kusto_client.close()
40 super().close()
41
42 def set_proxy(self, proxy_url: str):
43 self._kusto_client.set_proxy(proxy_url)
44
45 @distributed_trace(kind=SpanKind.CLIENT)
46 def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
47 """Ingest from local files.
48 :param file_descriptor: a FileDescriptor to be ingested.
49 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
50 """
51 file_descriptor = FileDescriptor.get_instance(file_descriptor)
52 IngestTracingAttributes.set_ingest_descriptor_attributes(file_descriptor, ingestion_properties)
53
54 super().ingest_from_file(file_descriptor, ingestion_properties)
55
56 stream_descriptor = StreamDescriptor.from_file_descriptor(file_descriptor)
57
58 with stream_descriptor.stream:
59 return self.ingest_from_stream(stream_descriptor, ingestion_properties)
60
61 @distributed_trace(kind=SpanKind.CLIENT)
62 def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
63 """Ingest from io streams.
64 :param azure.kusto.ingest.StreamDescriptor stream_descriptor: An object that contains a description of the stream to
65 be ingested.
66 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
67 """
68 stream_descriptor = StreamDescriptor.get_instance(stream_descriptor)
69 IngestTracingAttributes.set_ingest_descriptor_attributes(stream_descriptor, ingestion_properties)
70
71 super().ingest_from_stream(stream_descriptor, ingestion_properties)
72
73 return self._ingest_from_stream_with_client_request_id(stream_descriptor, ingestion_properties, None)
74
75 def _ingest_from_stream_with_client_request_id(
76 self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties, client_request_id: Optional[str]
77 ) -> IngestionResult:
78 stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
79 additional_properties = None
80 if client_request_id:
81 additional_properties = ClientRequestProperties()
82 additional_properties.client_request_id = client_request_id
83
84 self._kusto_client.execute_streaming_ingest(
85 ingestion_properties.database,
86 ingestion_properties.table,
87 stream_descriptor.stream,
88 None,
89 ingestion_properties.format.name,
90 additional_properties,
91 mapping_name=ingestion_properties.ingestion_mapping_reference,
92 )
93
94 return IngestionResult(IngestionStatus.SUCCESS, ingestion_properties.database, ingestion_properties.table, stream_descriptor.source_id)
95
96 def ingest_from_blob(
97 self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties, client_request_id: Optional[str] = None
98 ) -> IngestionResult:
99 IngestTracingAttributes.set_ingest_descriptor_attributes(blob_descriptor, ingestion_properties)
100 additional_properties = None
101 if client_request_id:
102 additional_properties = ClientRequestProperties()
103 additional_properties.client_request_id = client_request_id
104
105 self._kusto_client.execute_streaming_ingest(
106 ingestion_properties.database,
107 ingestion_properties.table,
108 None,
109 blob_descriptor.path,
110 ingestion_properties.format.name,
111 additional_properties,
112 mapping_name=ingestion_properties.ingestion_mapping_reference,
113 )
114 return IngestionResult(IngestionStatus.SUCCESS, ingestion_properties.database, ingestion_properties.table, blob_descriptor.source_id)