| 1 |
import uuid |
| 2 |
from io import SEEK_SET |
| 3 |
from typing import AnyStr, IO, TYPE_CHECKING, Union, Optional |
| 4 |
|
| 5 |
from azure.kusto.ingest.descriptors import DescriptorBase |
| 6 |
from tenacity import Retrying, _utils, stop_after_attempt, wait_random_exponential |
| 7 |
|
| 8 |
from azure.core.tracing.decorator import distributed_trace |
| 9 |
from azure.core.tracing import SpanKind |
| 10 |
|
| 11 |
from azure.kusto.data import KustoConnectionStringBuilder |
| 12 |
from azure.kusto.data.exceptions import KustoApiError, KustoClosedError |
| 13 |
from azure.kusto.data._telemetry import MonitoredActivity |
| 14 |
|
| 15 |
from . import BlobDescriptor, FileDescriptor, IngestionProperties, StreamDescriptor |
| 16 |
from ._ingest_telemetry import IngestTracingAttributes |
| 17 |
from ._stream_extensions import chain_streams, read_until_size_or_end |
| 18 |
from .base_ingest_client import BaseIngestClient, IngestionResult |
| 19 |
from .ingest_client import QueuedIngestClient |
| 20 |
from .streaming_ingest_client import KustoStreamingIngestClient |
| 21 |
|
| 22 |
if TYPE_CHECKING: |
| 23 |
pass |
| 24 |
|
| 25 |
|
| 26 |
class ManagedStreamingIngestClient(BaseIngestClient): |
| 27 |
""" |
| 28 |
Managed Streaming Ingestion Client. |
| 29 |
Will try to ingest with streaming, but if it fails, will fall back to queued ingestion. |
| 30 |
Each transient failure will be retried with exponential backoff. |
| 31 |
|
| 32 |
Managed streaming ingest client will fall back to queued if: |
| 33 |
- Multiple transient errors were encountered when trying to do streaming ingestion |
| 34 |
- The ingestion is too large for streaming ingestion (over 4MB) |
| 35 |
- The ingestion is directly from a blob |
| 36 |
""" |
| 37 |
|
| 38 |
MAX_STREAMING_SIZE_IN_BYTES = 4 * 1024 * 1024 |
| 39 |
|
| 40 |
def __init__( |
| 41 |
self, |
| 42 |
engine_kcsb: Union[KustoConnectionStringBuilder, str], |
| 43 |
dm_kcsb: Union[KustoConnectionStringBuilder, str, None] = None, |
| 44 |
auto_correct_endpoint: bool = True, |
| 45 |
): |
| 46 |
super().__init__() |
| 47 |
self.queued_client = QueuedIngestClient(dm_kcsb if dm_kcsb is not None else engine_kcsb, auto_correct_endpoint) |
| 48 |
self.streaming_client = KustoStreamingIngestClient(engine_kcsb, auto_correct_endpoint) |
| 49 |
self._set_retry_settings() |
| 50 |
|
| 51 |
def close(self) -> None: |
| 52 |
if not self._is_closed: |
| 53 |
self.queued_client.close() |
| 54 |
self.streaming_client.close() |
| 55 |
super().close() |
| 56 |
|
| 57 |
def _set_retry_settings(self, max_seconds_per_retry: float = _utils.MAX_WAIT, num_of_attempts: int = 3): |
| 58 |
self._num_of_attempts = num_of_attempts |
| 59 |
self._max_seconds_per_retry = max_seconds_per_retry |
| 60 |
|
| 61 |
def set_proxy(self, proxy_url: str): |
| 62 |
self.queued_client.set_proxy(proxy_url) |
| 63 |
self.streaming_client.set_proxy(proxy_url) |
| 64 |
|
| 65 |
@distributed_trace(kind=SpanKind.CLIENT) |
| 66 |
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 67 |
file_descriptor = FileDescriptor.get_instance(file_descriptor) |
| 68 |
IngestTracingAttributes.set_ingest_descriptor_attributes(file_descriptor, ingestion_properties) |
| 69 |
|
| 70 |
super().ingest_from_file(file_descriptor, ingestion_properties) |
| 71 |
|
| 72 |
stream_descriptor = StreamDescriptor.from_file_descriptor(file_descriptor) |
| 73 |
|
| 74 |
with stream_descriptor.stream: |
| 75 |
return self.ingest_from_stream(stream_descriptor, ingestion_properties) |
| 76 |
|
| 77 |
@distributed_trace(kind=SpanKind.CLIENT) |
| 78 |
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 79 |
stream_descriptor = StreamDescriptor.get_instance(stream_descriptor) |
| 80 |
IngestTracingAttributes.set_ingest_descriptor_attributes(stream_descriptor, ingestion_properties) |
| 81 |
|
| 82 |
super().ingest_from_stream(stream_descriptor, ingestion_properties) |
| 83 |
|
| 84 |
stream_descriptor = BaseIngestClient._prepare_stream(stream_descriptor, ingestion_properties) |
| 85 |
stream = stream_descriptor.stream |
| 86 |
|
| 87 |
buffered_stream = read_until_size_or_end(stream, self.MAX_STREAMING_SIZE_IN_BYTES + 1) |
| 88 |
length = len(buffered_stream.getbuffer()) |
| 89 |
|
| 90 |
stream_descriptor.stream = buffered_stream |
| 91 |
|
| 92 |
try: |
| 93 |
res = self._stream_with_retries(length, stream_descriptor, ingestion_properties) |
| 94 |
if res: |
| 95 |
return res |
| 96 |
stream_descriptor.stream = chain_streams([buffered_stream, stream]) |
| 97 |
except KustoApiError as ex: |
| 98 |
error = ex.get_api_error() |
| 99 |
if error.permanent: |
| 100 |
raise |
| 101 |
buffered_stream.seek(0, SEEK_SET) |
| 102 |
|
| 103 |
return self.queued_client.ingest_from_stream(stream_descriptor, ingestion_properties) |
| 104 |
|
| 105 |
@distributed_trace(kind=SpanKind.CLIENT) |
| 106 |
def ingest_from_blob(self, blob_descriptor: BlobDescriptor, ingestion_properties: IngestionProperties): |
| 107 |
""" |
| 108 |
Enqueue an ingest command from azure blobs. |
| 109 |
|
| 110 |
For ManagedStreamingIngestClient, this method always uses Queued Ingest, since it would be easier and faster to ingest blobs. |
| 111 |
|
| 112 |
To learn more about ingestion methods go to: |
| 113 |
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods |
| 114 |
:param azure.kusto.ingest.BlobDescriptor blob_descriptor: An object that contains a description of the blob to be ingested. |
| 115 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 116 |
""" |
| 117 |
IngestTracingAttributes.set_ingest_descriptor_attributes(blob_descriptor, ingestion_properties) |
| 118 |
|
| 119 |
if self._is_closed: |
| 120 |
raise KustoClosedError() |
| 121 |
blob_descriptor.fill_size() |
| 122 |
try: |
| 123 |
res = self._stream_with_retries(blob_descriptor.size, blob_descriptor, ingestion_properties) |
| 124 |
if res: |
| 125 |
return res |
| 126 |
except KustoApiError as ex: |
| 127 |
error = ex.get_api_error() |
| 128 |
if error.permanent: |
| 129 |
raise |
| 130 |
|
| 131 |
return self.queued_client.ingest_from_blob(blob_descriptor, ingestion_properties) |
| 132 |
|
| 133 |
def _stream_with_retries( |
| 134 |
self, |
| 135 |
length: int, |
| 136 |
descriptor: DescriptorBase, |
| 137 |
props: IngestionProperties, |
| 138 |
) -> Optional[IngestionResult]: |
| 139 |
from_stream = isinstance(descriptor, StreamDescriptor) |
| 140 |
if length > self.MAX_STREAMING_SIZE_IN_BYTES: |
| 141 |
return None |
| 142 |
for attempt in Retrying(stop=stop_after_attempt(self._num_of_attempts), wait=wait_random_exponential(max=self._max_seconds_per_retry), reraise=True): |
| 143 |
with attempt: |
| 144 |
client_request_id = ManagedStreamingIngestClient._get_request_id(descriptor.source_id, attempt.retry_state.attempt_number - 1) |
| 145 |
# trace attempt to ingest from stream |
| 146 |
if from_stream: |
| 147 |
descriptor.stream.seek(0, SEEK_SET) |
| 148 |
invoker = lambda: self.streaming_client._ingest_from_stream_with_client_request_id(descriptor, props, client_request_id) |
| 149 |
else: |
| 150 |
invoker = lambda: self.streaming_client.ingest_from_blob(descriptor, props, client_request_id) |
| 151 |
return MonitoredActivity.invoke( |
| 152 |
invoker, |
| 153 |
name_of_span="ManagedStreamingIngestClient.ingest_from_stream_attempt", |
| 154 |
tracing_attributes={"attemptNumber": attempt, "sourceIsStream": from_stream}, |
| 155 |
) |
| 156 |
|
| 157 |
@staticmethod |
| 158 |
def _get_request_id(source_id: uuid.UUID, attempt: int): |
| 159 |
return f"KPC.executeManagedStreamingIngest;{source_id};{attempt}" |