Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/base_ingest_client.py

raw
1 import gzip
2 import ipaddress
3 import os
4 import tempfile
5 import time
6 import uuid
7 from abc import ABCMeta, abstractmethod
8 from enum import Enum
9 from io import TextIOWrapper
10 from typing import TYPE_CHECKING, Union, IO, AnyStr, Optional, Tuple
11 from urllib.parse import urlparse
12
13 from azure.kusto.data.data_format import DataFormat, IngestionMappingKind
14 from azure.kusto.data.exceptions import KustoClosedError
15 from .descriptors import FileDescriptor, StreamDescriptor
16 from .ingestion_properties import IngestionProperties
17
18 if TYPE_CHECKING:
19 import pandas
20
21 INGEST_PREFIX = "ingest-"
22 PROTOCOL_SUFFIX = "://"
23
24
25 class IngestionStatus(Enum):
26 """
27 The ingestion was queued.
28 """
29
30 QUEUED = "QUEUED"
31 """
32 The ingestion was successfully streamed
33 """
34 SUCCESS = "SUCCESS"
35
36
37 class IngestionResult:
38 """
39 The result of an ingestion.
40 """
41
42 status: IngestionStatus
43 "Will be `Queued` if the ingestion is queued, or `Success` if the ingestion is streaming and successful."
44
45 database: str
46 "The name of the database where the ingestion was performed."
47
48 table: str
49 "The name of the table where the ingestion was performed."
50
51 source_id: uuid.UUID
52 "The source id of the ingestion."
53
54 blob_uri: Optional[str]
55 "The blob uri of the ingestion, if exists."
56
57 def __init__(self, status: IngestionStatus, database: str, table: str, source_id: uuid.UUID, blob_uri: Optional[str] = None):
58 self.status = status
59 self.database = database
60 self.table = table
61 self.source_id = source_id
62 self.blob_uri = blob_uri
63
64 def __repr__(self):
65 # Remove query parameters from blob_uri, if exists
66 obfuscated_path = None
67 if isinstance(self.blob_uri, str):
68 obfuscated_path = self.blob_uri.split("?")[0].split(";")[0]
69 blob_uri = f", obfuscated_blob_uri={obfuscated_path}" if obfuscated_path else ""
70 return f"IngestionResult(status={self.status}, database={self.database}, table={self.table}, source_id={self.source_id}{blob_uri})"
71
72
73 class BaseIngestClient(metaclass=ABCMeta):
74 def __init__(self):
75 self._is_closed: bool = False
76
77 def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult:
78 """Ingest from local files.
79 :param file_descriptor: a FileDescriptor to be ingested.
80 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
81 """
82 if self._is_closed:
83 raise KustoClosedError()
84
85 @abstractmethod
86 def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult:
87 """Ingest from io streams.
88 :param stream_descriptor: An object that contains a description of the stream to be ingested.
89 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
90 """
91 if self._is_closed:
92 raise KustoClosedError()
93
94 @abstractmethod
95 def set_proxy(self, proxy_url: str):
96 """Set proxy for the ingestion client.
97 :param str proxy_url: proxy url.
98 """
99 if self._is_closed:
100 raise KustoClosedError()
101
102 def ingest_from_dataframe(
103 self, df: "pandas.DataFrame", ingestion_properties: IngestionProperties, data_format: Optional[DataFormat] = None
104 ) -> IngestionResult:
105 """Enqueue an ingest command from local files.
106 To learn more about ingestion methods go to:
107 https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods
108 :param pandas.DataFrame df: input dataframe to ingest.
109 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
110 :param DataFormat data_format: Format to convert the dataframe to - Can be DataFormat.CSV, DataFormat.JSON or None. If not specified, it will try to infer it from the mapping, if not found, it will default to JSON.
111 """
112
113 if self._is_closed:
114 raise KustoClosedError()
115
116 from pandas import DataFrame
117
118 if not isinstance(df, DataFrame):
119 raise ValueError("Expected DataFrame instance, found {}".format(type(df)))
120
121 is_json = True
122
123 # If we are given CSV mapping, or the mapping format is explicitly set to CSV, we should use CSV
124 if not data_format:
125 if ingestion_properties is not None and (ingestion_properties.ingestion_mapping_type == IngestionMappingKind.CSV):
126 is_json = False
127 elif data_format == DataFormat.CSV:
128 is_json = False
129 elif data_format == DataFormat.JSON:
130 is_json = True
131 else:
132 raise ValueError("Unsupported format: {}. Supported formats are: CSV, JSON, None".format(data_format))
133
134 file_name = "df_{id}_{timestamp}_{uid}.{ext}.gz".format(id=id(df), timestamp=int(time.time()), uid=uuid.uuid4(), ext="json" if is_json else "csv")
135 temp_file_path = os.path.join(tempfile.gettempdir(), file_name)
136 with gzip.open(temp_file_path, "wb") as temp_file:
137 if is_json:
138 df.to_json(temp_file, orient="records", date_format="iso", lines=True)
139 ingestion_properties.format = DataFormat.JSON
140 else:
141 df.to_csv(temp_file, index=False, encoding="utf-8", header=False)
142 ingestion_properties.ignore_first_record = False
143 ingestion_properties.format = DataFormat.CSV
144
145 try:
146 return self.ingest_from_file(temp_file_path, ingestion_properties)
147 finally:
148 os.unlink(temp_file_path)
149
150 @staticmethod
151 def _prepare_stream(stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> StreamDescriptor:
152 """
153 Prepares a StreamDescriptor instance for ingest operation based on ingestion properties
154 :param StreamDescriptor stream_descriptor: Stream descriptor instance
155 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
156 :return prepared stream descriptor
157 """
158 new_descriptor = StreamDescriptor.get_instance(stream_descriptor)
159
160 if isinstance(new_descriptor.stream, TextIOWrapper):
161 new_descriptor.stream = new_descriptor.stream.buffer
162
163 should_compress = BaseIngestClient._should_compress(new_descriptor, ingestion_properties)
164 if should_compress:
165 new_descriptor.compress_stream()
166
167 return new_descriptor
168
169 @staticmethod
170 def _prepare_file(file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> Tuple[FileDescriptor, bool]:
171 """
172 Prepares a FileDescriptor instance for ingest operation based on ingestion properties
173 :param FileDescriptor file_descriptor: File descriptor instance
174 :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
175 :return prepared file descriptor
176 """
177 descriptor = FileDescriptor.get_instance(file_descriptor)
178
179 should_compress = BaseIngestClient._should_compress(descriptor, ingestion_properties)
180 return descriptor, should_compress
181
182 @staticmethod
183 def _should_compress(new_descriptor: Union[FileDescriptor, StreamDescriptor], ingestion_properties: IngestionProperties) -> bool:
184 """
185 Checks if descriptor should be compressed based on ingestion properties and current format
186 """
187 return not new_descriptor.is_compressed and ingestion_properties.format.compressible
188
189 def close(self) -> None:
190 self._is_closed = True
191
192 def __enter__(self):
193 return self
194
195 def __exit__(self, exc_type, exc_val, exc_tb):
196 self.close()
197
198 @staticmethod
199 def get_ingestion_endpoint(cluster_url: str) -> str:
200 if INGEST_PREFIX in cluster_url or not cluster_url or BaseIngestClient.is_reserved_hostname(cluster_url):
201 return cluster_url
202 else:
203 return cluster_url.replace(PROTOCOL_SUFFIX, PROTOCOL_SUFFIX + INGEST_PREFIX, 1)
204
205 @staticmethod
206 def get_query_endpoint(cluster_url: str) -> str:
207 if INGEST_PREFIX in cluster_url:
208 return cluster_url.replace(INGEST_PREFIX, "", 1)
209 else:
210 return cluster_url
211
212 @staticmethod
213 def is_reserved_hostname(raw_uri: str) -> bool:
214 url = urlparse(raw_uri)
215 if not url.netloc:
216 return True
217 authority = url.netloc.split(":")[0] # removes port if exists
218 try:
219 is_ip = ipaddress.ip_address(authority)
220 except ValueError:
221 is_ip = False
222 is_localhost = "localhost" in authority
223 return is_localhost or is_ip or authority.lower() == "onebox.dev.kusto.windows.net"