Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/descriptors.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import abc
4 import os
5 import random
6 import shutil
7 import struct
8 import uuid
9 from copy import copy
10 from gzip import GzipFile
11 from io import BytesIO, SEEK_END
12 from typing import Union, Optional, AnyStr, IO, List, Dict
13 from zipfile import ZipFile
14
15 from azure.storage.blob import BlobClient
16
17 OptionalUUID = Optional[Union[str, uuid.UUID]]
18
19
20 def ensure_uuid(maybe_uuid: OptionalUUID) -> uuid.UUID:
21 if not maybe_uuid:
22 return uuid.uuid4()
23
24 if type(maybe_uuid) == uuid.UUID:
25 return maybe_uuid
26
27 return uuid.UUID(f"{maybe_uuid}", version=4)
28
29
30 class DescriptorBase(abc.ABC):
31 """This base class abstracts tracing attributes for all implementations."""
32
33 _SOURCE_ID = "source_id"
34
35 @abc.abstractmethod
36 def get_tracing_attributes(self) -> dict:
37 """Gets dictionary of attributes to be documented during tracing"""
38 pass
39
40
41 class FileDescriptor(DescriptorBase):
42 """FileDescriptor is used to describe a file that will be used as an ingestion source."""
43
44 # Gzip keeps the decompressed stream size as a UINT32 in the last 4 bytes of the stream, however this poses a limit to the expressed size which is 4GB
45 # The standard says that when the size is bigger than 4GB, the UINT rolls over.
46 # The below constant expresses the maximal size of a compressed stream that will not cause the UINT32 to rollover given a maximal compression ratio of 1:40
47 GZIP_MAX_DISK_SIZE_FOR_DETECTION = int(4 * 1024 * 1024 * 1024 / 40)
48 DEFAULT_COMPRESSION_RATIO = 11
49 _FILE_PATH = "file_path"
50
51 def __init__(self, path: str, size: Optional[int] = None, source_id: OptionalUUID = None):
52 """
53 :param path: file path.
54 :type path: str.
55 :param size: estimated size of file if known. if None or 0 will try to guess.
56 :type size: Optional[int].
57 :param source_id: a v4 uuid to serve as the source's id.
58 :type source_id: OptionalUUID
59 """
60 self.path: str = path
61 self._size: Optional[int] = size
62 self._detect_size_once: bool = not size
63
64 self.source_id: uuid.UUID = ensure_uuid(source_id)
65 self.stream_name: str = os.path.basename(self.path)
66
67 @property
68 def size(self) -> int:
69 if self._detect_size_once:
70 self._detect_size()
71 self._detect_size_once = False
72
73 return self._size
74
75 @size.setter
76 def size(self, size: int):
77 if size:
78 self._size = size
79 self._detect_size_once = False
80
81 def _detect_size(self):
82 uncompressed_size = 0
83 if self.path.endswith(".gz"):
84 # This logic follow after the C# implementation
85 # See IngstionHelpers.cs for an explanation as to what stands behind it
86 with open(self.path, "rb") as f:
87 disk_size = f.seek(-4, SEEK_END)
88 uncompressed_size = struct.unpack("I", f.read(4))[0]
89 if (disk_size >= uncompressed_size) or (disk_size >= self.GZIP_MAX_DISK_SIZE_FOR_DETECTION):
90 uncompressed_size = disk_size * self.DEFAULT_COMPRESSION_RATIO
91
92 elif self.path.endswith(".zip"):
93 with ZipFile(self.path) as zip_archive:
94 for f in zip_archive.infolist():
95 uncompressed_size += f.file_size
96
97 else:
98 uncompressed_size = os.path.getsize(self.path)
99
100 self._size = uncompressed_size
101
102 @property
103 def is_compressed(self) -> bool:
104 return self.path.endswith(".gz") or self.path.endswith(".zip")
105
106 def open(self, should_compress: bool) -> BytesIO:
107 if should_compress:
108 file_stream = self.compress_stream()
109 else:
110 file_stream = open(self.path, "rb")
111 return file_stream
112
113 def compress_stream(self) -> BytesIO:
114 self.stream_name += ".gz"
115 file_stream = BytesIO()
116 with open(self.path, "rb") as f_in, GzipFile(filename="data", fileobj=file_stream, mode="wb") as f_out:
117 shutil.copyfileobj(f_in, f_out)
118 file_stream.seek(0)
119 return file_stream
120
121 def get_tracing_attributes(self) -> dict:
122 return {self._FILE_PATH: self.stream_name, self._SOURCE_ID: str(self.source_id)}
123
124 @classmethod
125 def get_instance(cls, file_descriptor: Union["FileDescriptor", str]) -> "FileDescriptor":
126 if not isinstance(file_descriptor, cls):
127 return cls(file_descriptor)
128 return file_descriptor
129
130
131 class BlobDescriptor(DescriptorBase):
132 """BlobDescriptor is used to describe a blob that will be used as an ingestion source"""
133
134 _BLOB_URI = "blob_uri"
135
136 def __init__(self, path: str, size: Optional[int] = None, source_id: OptionalUUID = None):
137 """
138 :param path: blob uri.
139 :type path: str.
140 :param size: estimated size of file if known.
141 :type size: Optional[int].
142 :param source_id: a v4 uuid to serve as the source's id.
143 :type source_id: OptionalUUID
144 """
145 self.path: str = path
146 self.size: Optional[int] = size
147 self.source_id: uuid.UUID = ensure_uuid(source_id)
148
149 def get_tracing_attributes(self) -> dict:
150 # Remove query parameters from self.path, if exists
151 if self.path:
152 obfuscated_path = self.path.split("?")[0].split(";")[0]
153 return {self._BLOB_URI: obfuscated_path, self._SOURCE_ID: str(self.source_id)}
154
155 def fill_size(self):
156 if not self.size:
157 self.size = BlobClient.from_blob_url(self.path).get_blob_properties().size
158
159
160 class StreamDescriptor(DescriptorBase):
161 """StreamDescriptor is used to describe a stream that will be used as ingestion source"""
162
163 _STREAM_NAME = "stream_name"
164
165 # TODO: currently we always assume that streams are gz compressed (will get compressed before sending), should we expand that?
166 def __init__(
167 self, stream: IO[AnyStr], source_id: OptionalUUID = None, is_compressed: bool = False, stream_name: Optional[str] = None, size: Optional[int] = None
168 ):
169 """
170 :param stream: in-memory stream object.
171 :type stream: io.BaseIO
172 :param source_id: a v4 uuid to serve as the sources id.
173 :type source_id: OptionalUUID
174 :param is_compressed: specify if the provided stream is compressed
175 :type is_compressed: boolean
176 """
177 self.stream: IO[AnyStr] = stream
178 self.source_id: uuid.UUID = ensure_uuid(source_id)
179 self.is_compressed: bool = is_compressed
180 self.stream_name: str = stream_name
181 if self.stream_name is None:
182 self.stream_name = "stream"
183 if is_compressed:
184 self.stream_name += ".gz"
185 self.size: Optional[int] = size
186
187 def compress_stream(self) -> None:
188 stream = self.stream
189 zipped_stream = BytesIO()
190 stream_buffer = stream.read()
191 with GzipFile(filename="data", fileobj=zipped_stream, mode="wb") as f_out:
192 if isinstance(stream_buffer, str):
193 data = bytes(stream_buffer, "utf-8")
194 f_out.write(data)
195 else:
196 f_out.write(stream_buffer)
197 zipped_stream.seek(0)
198 self.is_compressed = True
199 self.stream_name += ".gz"
200 self.stream = zipped_stream
201
202 @staticmethod
203 def from_file_descriptor(file_descriptor: Union[FileDescriptor, str]) -> "StreamDescriptor":
204 """
205 Transforms FileDescriptor instance into StreamDescriptor instance. Note that stream is open when instance is returned
206 :param Union[FileDescriptor, str] file_descriptor: File Descriptor instance
207 :return new StreamDescriptor instance
208 """
209 descriptor = FileDescriptor.get_instance(file_descriptor)
210 stream = open(descriptor.path, "rb")
211 is_compressed = descriptor.path.endswith(".gz") or descriptor.path.endswith(".zip")
212 stream_descriptor = StreamDescriptor(stream, descriptor.source_id, is_compressed, descriptor.stream_name, descriptor.size)
213 return stream_descriptor
214
215 @classmethod
216 def get_instance(cls, stream_descriptor: Union["StreamDescriptor", IO[AnyStr]]) -> "StreamDescriptor":
217 if not isinstance(stream_descriptor, cls):
218 descriptor = cls(stream_descriptor)
219 else:
220 descriptor = copy(stream_descriptor)
221 return descriptor
222
223 def get_tracing_attributes(self) -> dict:
224 return {self._STREAM_NAME: self.stream_name, self._SOURCE_ID: str(self.source_id)}