Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/managed_streaming_ingest_client.py

raw
1 import uuid
2 from io import SEEK_SET
3 from typing import AnyStr, IO, TYPE_CHECKING, Union, Optional
4
5 from azure.kusto.ingest.descriptors import DescriptorBase
6 from tenacity import Retrying, _utils, stop_after_attempt, wait_random_exponential
7
8 from azure.core.tracing.decorator import distributed_trace
9 from azure.core.tracing import SpanKind
10
11 from azure.kusto.data import KustoConnectionStringBuilder
12 from azure.kusto.data.exceptions import KustoApiError, KustoClosedError
13 from azure.kusto.data._telemetry import MonitoredActivity
14
15 from . import BlobDescriptor, FileDescriptor, IngestionProperties, StreamDescriptor
16 from ._ingest_telemetry import IngestTracingAttributes
17 from ._stream_extensions import chain_streams, read_until_size_or_end
18 from .base_ingest_client import BaseIngestClient, IngestionResult
19 from .ingest_client import QueuedIngestClient
20 from .streaming_ingest_client import KustoStreamingIngestClient
21
22 if TYPE_CHECKING:
23 pass
24
25
26 class ManagedStreamingIngestClient(BaseIngestClient):
27 """
28 Managed Streaming Ingestion Client.
29 Will try to ingest with streaming, but if it fails, will fall back to queued ingestion.
30 Each transient failure will be retried with exponential backoff.
31
32 Managed streaming ingest client will fall back to queued if:
33 - Multiple transient errors were encountered when trying to do streaming ingestion
34 - The ingestion is too large for streaming ingestion (over 4MB)
35 - The ingestion is directly from a blob
36 """
37
38 MAX_STREAMING_SIZE_IN_BYTES = 4 * 1024 * 1024
39
40 def __init__(
41 self,
42 engine_kcsb: Union[KustoConnectionStringBuilder, str],
43 dm_kcsb: Union[KustoConnectionStringBuilder, str, None] = None,
44 auto_correct_endpoint: bool = True,
45 ):
46 super().__init__()
47 self.queued_client = QueuedIngestClient(dm_kcsb if dm_kcsb is not None else engine_kcsb, auto_correct_endpoint)
48 self.streaming_client = KustoStreamingIngestClient(engine_kcsb, auto_correct_endpoint)
49 self._set_retry_settings()
50
51 def close(self) -> None:
52 if not self._is_closed:
53 self.queued_client.close()
54 self.streaming_client.close()
55 super().close()
56
57 def _set_retry_settings(self, max_seconds_per_retry: float = _utils.MAX_WAIT, num_of_attempts: int = 3):
58 self._num_of_attempts = num_of_attempts
59 self._max_seconds_per_retry = max_seconds_per_retry
60
61 def set_proxy(self, proxy_url: str):
62 self.queued_client.set_proxy(proxy_url)
63 self.streaming_client.set_proxy(proxy_url)
64
65 @distributed_trace(kind=SpanKind.CLIENT)
66 def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
67 file_descriptor = FileDescriptor.get_instance(file_descriptor)
68 IngestTracingAttributes.set_ingest_descriptor_attributes(file_descriptor, ingestion_properties)
69
70 super().ingest_from_file(file_descriptor, ingestion_properties)
71
72 stream_descriptor = StreamDescriptor.from_file_descriptor(file_descriptor)
73
74 with stream_descriptor.stream:
75 return self.ingest_from_stream(stream_descriptor, ingestion_properties)
76
77 @distributed_trace(kind=SpanKind.CLIENT)
78 def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
79 stream_descriptor = StreamDescriptor.get_instance(stream_descriptor)
80 IngestTracingAttributes.set_ingest_descriptor_attributes(stream_descriptor, ingestion_properties)
81
82 super().ingest_from_stream(stream_descriptor, ingestion_properties)
83
84 stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties)
85 stream = stream_descriptor.stream
86
87 buffered_stream = read_until_size_or_end(stream, self.MAX_STREAMING_SIZE_IN_BYTES + 1)
88 length = len(buffered_stream.getbuffer())
89
90 stream_descriptor.stream = buffered_stream
91
92 try:
93 res = self._stream_with_retries(length, stream_descriptor, ingestion_properties)
94 if res:
95 return res
96 stream_descriptor.stream = chain_streams([buffered_stream, stream])
97 except KustoApiError as ex:
98 error = ex.get_api_error()
99 if error.permanent:
100 raise
101 buffered_stream.seek(0, SEEK_SET)
102
103 return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties)
104
105 @distributed_trace(kind=SpanKind.CLIENT)
106 def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties):
107 """
108 Enqueue an ingest command from azure blobs.
109
110 For ManagedStreamingIngestClient, this method always uses Queued Ingest, since it would be easier and faster to ingest blobs.
111
112 To learn more about ingestion methods go to:
113 https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
114 :param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested.
115 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
116 """
117 IngestTracingAttributes.set_ingest_descriptor_attributes(blob_descriptor, ingestion_properties)
118
119 if self._is_closed:
120 raise KustoClosedError()
121 blob_descriptor.fill_size()
122 try:
123 res = self._stream_with_retries(blob_descriptor.size, blob_descriptor, ingestion_properties)
124 if res:
125 return res
126 except KustoApiError as ex:
127 error = ex.get_api_error()
128 if error.permanent:
129 raise
130
131 return self.queued_client.ingest_from_blob(blob_descriptor, ingestion_properties)
132
133 def _stream_with_retries(
134 self,
135 length: int,
136 descriptor: DescriptorBase,
137 props: IngestionProperties,
138 ) -> Optional[IngestionResult]:
139 from_stream = isinstance(descriptor, StreamDescriptor)
140 if length > self.MAX_STREAMING_SIZE_IN_BYTES:
141 return None
142 for attempt in Retrying(stop=stop_after_attempt(self._num_of_attempts), wait=wait_random_exponential(max=self._max_seconds_per_retry), reraise=True):
143 with attempt:
144 client_request_id = ManagedStreamingIngestClient._get_request_id(descriptor.source_id, attempt.retry_state.attempt_number - 1)
145 # trace attempt to ingest from stream
146 if from_stream:
147 descriptor.stream.seek(0, SEEK_SET)
148 invoker = lambda: self.streaming_client._ingest_from_stream_with_client_request_id(descriptor, props, client_request_id)
149 else:
150 invoker = lambda: self.streaming_client.ingest_from_blob(descriptor, props, client_request_id)
151 return MonitoredActivity.invoke(
152 invoker,
153 name_of_span="ManagedStreamingIngestClient.ingest_from_stream_attempt",
154 tracing_attributes={"attemptNumber": attempt, "sourceIsStream": from_stream},
155 )
156
157 @staticmethod
158 def _get_request_id(source_id: uuid.UUID, attempt: int):
159 return f"KPC.executeManagedStreamingIngest;{source_id};{attempt}"