| 1 |
import gzip |
| 2 |
import ipaddress |
| 3 |
import os |
| 4 |
import tempfile |
| 5 |
import time |
| 6 |
import uuid |
| 7 |
from abc import ABCMeta, abstractmethod |
| 8 |
from enum import Enum |
| 9 |
from io import TextIOWrapper |
| 10 |
from typing import TYPE_CHECKING, Union, IO, AnyStr, Optional, Tuple |
| 11 |
from urllib.parse import urlparse |
| 12 |
|
| 13 |
from azure.kusto.data.data_format import DataFormat, IngestionMappingKind |
| 14 |
from azure.kusto.data.exceptions import KustoClosedError |
| 15 |
from .descriptors import FileDescriptor, StreamDescriptor |
| 16 |
from .ingestion_properties import IngestionProperties |
| 17 |
|
| 18 |
if TYPE_CHECKING: |
| 19 |
import pandas |
| 20 |
|
| 21 |
INGEST_PREFIX = "ingest-" |
| 22 |
PROTOCOL_SUFFIX = "://" |
| 23 |
|
| 24 |
|
| 25 |
class IngestionStatus(Enum): |
| 26 |
""" |
| 27 |
The ingestion was queued. |
| 28 |
""" |
| 29 |
|
| 30 |
QUEUED = "QUEUED" |
| 31 |
""" |
| 32 |
The ingestion was successfully streamed |
| 33 |
""" |
| 34 |
SUCCESS = "SUCCESS" |
| 35 |
|
| 36 |
|
| 37 |
class IngestionResult: |
| 38 |
""" |
| 39 |
The result of an ingestion. |
| 40 |
""" |
| 41 |
|
| 42 |
status: IngestionStatus |
| 43 |
"Will be `Queued` if the ingestion is queued, or `Success` if the ingestion is streaming and successful." |
| 44 |
|
| 45 |
database: str |
| 46 |
"The name of the database where the ingestion was performed." |
| 47 |
|
| 48 |
table: str |
| 49 |
"The name of the table where the ingestion was performed." |
| 50 |
|
| 51 |
source_id: uuid.UUID |
| 52 |
"The source id of the ingestion." |
| 53 |
|
| 54 |
blob_uri: Optional[str] |
| 55 |
"The blob uri of the ingestion, if exists." |
| 56 |
|
| 57 |
def __init__(self, status: IngestionStatus, database: str, table: str, source_id: uuid.UUID, blob_uri: Optional[str] = None): |
| 58 |
self.status = status |
| 59 |
self.database = database |
| 60 |
self.table = table |
| 61 |
self.source_id = source_id |
| 62 |
self.blob_uri = blob_uri |
| 63 |
|
| 64 |
def __repr__(self): |
| 65 |
# Remove query parameters from blob_uri, if exists |
| 66 |
obfuscated_path = None |
| 67 |
if isinstance(self.blob_uri, str): |
| 68 |
obfuscated_path = self.blob_uri.split("?")[0].split(";")[0] |
| 69 |
blob_uri = f", obfuscated_blob_uri={obfuscated_path}" if obfuscated_path else "" |
| 70 |
return f"IngestionResult(status={self.status}, database={self.database}, table={self.table}, source_id={self.source_id}{blob_uri})" |
| 71 |
|
| 72 |
|
| 73 |
class BaseIngestClient(metaclass=ABCMeta): |
| 74 |
def __init__(self): |
| 75 |
self._is_closed: bool = False |
| 76 |
|
| 77 |
def ingest_from_file(self, file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 78 |
"""Ingest from local files. |
| 79 |
:param file_descriptor: a FileDescriptor to be ingested. |
| 80 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 81 |
""" |
| 82 |
if self._is_closed: |
| 83 |
raise KustoClosedError() |
| 84 |
|
| 85 |
@abstractmethod |
| 86 |
def ingest_from_stream(self, stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> IngestionResult: |
| 87 |
"""Ingest from io streams. |
| 88 |
:param stream_descriptor: An object that contains a description of the stream to be ingested. |
| 89 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 90 |
""" |
| 91 |
if self._is_closed: |
| 92 |
raise KustoClosedError() |
| 93 |
|
| 94 |
@abstractmethod |
| 95 |
def set_proxy(self, proxy_url: str): |
| 96 |
"""Set proxy for the ingestion client. |
| 97 |
:param str proxy_url: proxy url. |
| 98 |
""" |
| 99 |
if self._is_closed: |
| 100 |
raise KustoClosedError() |
| 101 |
|
| 102 |
def ingest_from_dataframe( |
| 103 |
self, df: "pandas.DataFrame", ingestion_properties: IngestionProperties, data_format: Optional[DataFormat] = None |
| 104 |
) -> IngestionResult: |
| 105 |
"""Enqueue an ingest command from local files. |
| 106 |
To learn more about ingestion methods go to: |
| 107 |
https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview#ingestion-methods |
| 108 |
:param pandas.DataFrame df: input dataframe to ingest. |
| 109 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 110 |
:param DataFormat data_format: Format to convert the dataframe to - Can be DataFormat.CSV, DataFormat.JSON or None. If not specified, it will try to infer it from the mapping, if not found, it will default to JSON. |
| 111 |
""" |
| 112 |
|
| 113 |
if self._is_closed: |
| 114 |
raise KustoClosedError() |
| 115 |
|
| 116 |
from pandas import DataFrame |
| 117 |
|
| 118 |
if not isinstance(df, DataFrame): |
| 119 |
raise ValueError("Expected DataFrame instance, found {}".format(type(df))) |
| 120 |
|
| 121 |
is_json = True |
| 122 |
|
| 123 |
# If we are given CSV mapping, or the mapping format is explicitly set to CSV, we should use CSV |
| 124 |
if not data_format: |
| 125 |
if ingestion_properties is not None and (ingestion_properties.ingestion_mapping_type == IngestionMappingKind.CSV): |
| 126 |
is_json = False |
| 127 |
elif data_format == DataFormat.CSV: |
| 128 |
is_json = False |
| 129 |
elif data_format == DataFormat.JSON: |
| 130 |
is_json = True |
| 131 |
else: |
| 132 |
raise ValueError("Unsupported format: {}. Supported formats are: CSV, JSON, None".format(data_format)) |
| 133 |
|
| 134 |
file_name = "df_{id}_{timestamp}_{uid}.{ext}.gz".format(id=id(df), timestamp=int(time.time()), uid=uuid.uuid4(), ext="json" if is_json else "csv") |
| 135 |
temp_file_path = os.path.join(tempfile.gettempdir(), file_name) |
| 136 |
with gzip.open(temp_file_path, "wb") as temp_file: |
| 137 |
if is_json: |
| 138 |
df.to_json(temp_file, orient="records", date_format="iso", lines=True) |
| 139 |
ingestion_properties.format = DataFormat.JSON |
| 140 |
else: |
| 141 |
df.to_csv(temp_file, index=False, encoding="utf-8", header=False) |
| 142 |
ingestion_properties.ignore_first_record = False |
| 143 |
ingestion_properties.format = DataFormat.CSV |
| 144 |
|
| 145 |
try: |
| 146 |
return self.ingest_from_file(temp_file_path, ingestion_properties) |
| 147 |
finally: |
| 148 |
os.unlink(temp_file_path) |
| 149 |
|
| 150 |
@staticmethod |
| 151 |
def _prepare_stream(stream_descriptor: Union[StreamDescriptor, IO[AnyStr]], ingestion_properties: IngestionProperties) -> StreamDescriptor: |
| 152 |
""" |
| 153 |
Prepares a StreamDescriptor instance for ingest operation based on ingestion properties |
| 154 |
:param StreamDescriptor stream_descriptor: Stream descriptor instance |
| 155 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 156 |
:return prepared stream descriptor |
| 157 |
""" |
| 158 |
new_descriptor = StreamDescriptor.get_instance(stream_descriptor) |
| 159 |
|
| 160 |
if isinstance(new_descriptor.stream, TextIOWrapper): |
| 161 |
new_descriptor.stream = new_descriptor.stream.buffer |
| 162 |
|
| 163 |
should_compress = BaseIngestClient._should_compress(new_descriptor, ingestion_properties) |
| 164 |
if should_compress: |
| 165 |
new_descriptor.compress_stream() |
| 166 |
|
| 167 |
return new_descriptor |
| 168 |
|
| 169 |
@staticmethod |
| 170 |
def _prepare_file(file_descriptor: Union[FileDescriptor, str], ingestion_properties: IngestionProperties) -> Tuple[FileDescriptor, bool]: |
| 171 |
""" |
| 172 |
Prepares a FileDescriptor instance for ingest operation based on ingestion properties |
| 173 |
:param FileDescriptor file_descriptor: File descriptor instance |
| 174 |
:param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties. |
| 175 |
:return prepared file descriptor |
| 176 |
""" |
| 177 |
descriptor = FileDescriptor.get_instance(file_descriptor) |
| 178 |
|
| 179 |
should_compress = BaseIngestClient._should_compress(descriptor, ingestion_properties) |
| 180 |
return descriptor, should_compress |
| 181 |
|
| 182 |
@staticmethod |
| 183 |
def _should_compress(new_descriptor: Union[FileDescriptor, StreamDescriptor], ingestion_properties: IngestionProperties) -> bool: |
| 184 |
""" |
| 185 |
Checks if descriptor should be compressed based on ingestion properties and current format |
| 186 |
""" |
| 187 |
return not new_descriptor.is_compressed and ingestion_properties.format.compressible |
| 188 |
|
| 189 |
def close(self) -> None: |
| 190 |
self._is_closed = True |
| 191 |
|
| 192 |
def __enter__(self): |
| 193 |
return self |
| 194 |
|
| 195 |
def __exit__(self, exc_type, exc_val, exc_tb): |
| 196 |
self.close() |
| 197 |
|
| 198 |
@staticmethod |
| 199 |
def get_ingestion_endpoint(cluster_url: str) -> str: |
| 200 |
if INGEST_PREFIX in cluster_url or not cluster_url or BaseIngestClient.is_reserved_hostname(cluster_url): |
| 201 |
return cluster_url |
| 202 |
else: |
| 203 |
return cluster_url.replace(PROTOCOL_SUFFIX, PROTOCOL_SUFFIX + INGEST_PREFIX, 1) |
| 204 |
|
| 205 |
@staticmethod |
| 206 |
def get_query_endpoint(cluster_url: str) -> str: |
| 207 |
if INGEST_PREFIX in cluster_url: |
| 208 |
return cluster_url.replace(INGEST_PREFIX, "", 1) |
| 209 |
else: |
| 210 |
return cluster_url |
| 211 |
|
| 212 |
@staticmethod |
| 213 |
def is_reserved_hostname(raw_uri: str) -> bool: |
| 214 |
url = urlparse(raw_uri) |
| 215 |
if not url.netloc: |
| 216 |
return True |
| 217 |
authority = url.netloc.split(":")[0] # removes port if exists |
| 218 |
try: |
| 219 |
is_ip = ipaddress.ip_address(authority) |
| 220 |
except ValueError: |
| 221 |
is_ip = False |
| 222 |
is_localhost = "localhost" in authority |
| 223 |
return is_localhost or is_ip or authority.lower() == "onebox.dev.kusto.windows.net" |