| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
import abc |
| 4 |
import os |
| 5 |
import random |
| 6 |
import shutil |
| 7 |
import struct |
| 8 |
import uuid |
| 9 |
from copy import copy |
| 10 |
from gzip import GzipFile |
| 11 |
from io import BytesIO, SEEK_END |
| 12 |
from typing import Union, Optional, AnyStr, IO, List, Dict |
| 13 |
from zipfile import ZipFile |
| 14 |
|
| 15 |
from azure.storage.blob import BlobClient |
| 16 |
|
| 17 |
OptionalUUID = Optional[Union[str, uuid.UUID]] |
| 18 |
|
| 19 |
|
| 20 |
def ensure_uuid(maybe_uuid: OptionalUUID) -> uuid.UUID: |
| 21 |
if not maybe_uuid: |
| 22 |
return uuid.uuid4() |
| 23 |
|
| 24 |
if type(maybe_uuid) == uuid.UUID: |
| 25 |
return maybe_uuid |
| 26 |
|
| 27 |
return uuid.UUID(f"{maybe_uuid}", version=4) |
| 28 |
|
| 29 |
|
| 30 |
class DescriptorBase(abc.ABC): |
| 31 |
"""This base class abstracts tracing attributes for all implementations.""" |
| 32 |
|
| 33 |
_SOURCE_ID = "source_id" |
| 34 |
|
| 35 |
@abc.abstractmethod |
| 36 |
def get_tracing_attributes(self) -> dict: |
| 37 |
"""Gets dictionary of attributes to be documented during tracing""" |
| 38 |
pass |
| 39 |
|
| 40 |
|
| 41 |
class FileDescriptor(DescriptorBase): |
| 42 |
"""FileDescriptor is used to describe a file that will be used as an ingestion source.""" |
| 43 |
|
| 44 |
# Gzip keeps the decompressed stream size as a UINT32 in the last 4 bytes of the stream, however this poses a limit to the expressed size which is 4GB |
| 45 |
# The standard says that when the size is bigger than 4GB, the UINT rolls over. |
| 46 |
# The below constant expresses the maximal size of a compressed stream that will not cause the UINT32 to rollover given a maximal compression ratio of 1:40 |
| 47 |
GZIP_MAX_DISK_SIZE_FOR_DETECTION = int(4 * 1024 * 1024 * 1024 / 40) |
| 48 |
DEFAULT_COMPRESSION_RATIO = 11 |
| 49 |
_FILE_PATH = "file_path" |
| 50 |
|
| 51 |
def __init__(self, path: str, size: Optional[int] = None, source_id: OptionalUUID = None): |
| 52 |
""" |
| 53 |
:param path: file path. |
| 54 |
:type path: str. |
| 55 |
:param size: estimated size of file if known. if None or 0 will try to guess. |
| 56 |
:type size: Optional[int]. |
| 57 |
:param source_id: a v4 uuid to serve as the source's id. |
| 58 |
:type source_id: OptionalUUID |
| 59 |
""" |
| 60 |
self.path: str = path |
| 61 |
self._size: Optional[int] = size |
| 62 |
self._detect_size_once: bool = not size |
| 63 |
|
| 64 |
self.source_id: uuid.UUID = ensure_uuid(source_id) |
| 65 |
self.stream_name: str = os.path.basename(self.path) |
| 66 |
|
| 67 |
@property |
| 68 |
def size(self) -> int: |
| 69 |
if self._detect_size_once: |
| 70 |
self._detect_size() |
| 71 |
self._detect_size_once = False |
| 72 |
|
| 73 |
return self._size |
| 74 |
|
| 75 |
@size.setter |
| 76 |
def size(self, size: int): |
| 77 |
if size: |
| 78 |
self._size = size |
| 79 |
self._detect_size_once = False |
| 80 |
|
| 81 |
def _detect_size(self): |
| 82 |
uncompressed_size = 0 |
| 83 |
if self.path.endswith(".gz"): |
| 84 |
# This logic follow after the C# implementation |
| 85 |
# See IngstionHelpers.cs for an explanation as to what stands behind it |
| 86 |
with open(self.path, "rb") as f: |
| 87 |
disk_size = f.seek(-4, SEEK_END) |
| 88 |
uncompressed_size = struct.unpack("I", f.read(4))[0] |
| 89 |
if (disk_size >= uncompressed_size) or (disk_size >= self.GZIP_MAX_DISK_SIZE_FOR_DETECTION): |
| 90 |
uncompressed_size = disk_size * self.DEFAULT_COMPRESSION_RATIO |
| 91 |
|
| 92 |
elif self.path.endswith(".zip"): |
| 93 |
with ZipFile(self.path) as zip_archive: |
| 94 |
for f in zip_archive.infolist(): |
| 95 |
uncompressed_size += f.file_size |
| 96 |
|
| 97 |
else: |
| 98 |
uncompressed_size = os.path.getsize(self.path) |
| 99 |
|
| 100 |
self._size = uncompressed_size |
| 101 |
|
| 102 |
@property |
| 103 |
def is_compressed(self) -> bool: |
| 104 |
return self.path.endswith(".gz") or self.path.endswith(".zip") |
| 105 |
|
| 106 |
def open(self, should_compress: bool) -> BytesIO: |
| 107 |
if should_compress: |
| 108 |
file_stream = self.compress_stream() |
| 109 |
else: |
| 110 |
file_stream = open(self.path, "rb") |
| 111 |
return file_stream |
| 112 |
|
| 113 |
def compress_stream(self) -> BytesIO: |
| 114 |
self.stream_name += ".gz" |
| 115 |
file_stream = BytesIO() |
| 116 |
with open(self.path, "rb") as f_in, GzipFile(filename="data", fileobj=file_stream, mode="wb") as f_out: |
| 117 |
shutil.copyfileobj(f_in, f_out) |
| 118 |
file_stream.seek(0) |
| 119 |
return file_stream |
| 120 |
|
| 121 |
def get_tracing_attributes(self) -> dict: |
| 122 |
return {self._FILE_PATH: self.stream_name, self._SOURCE_ID: str(self.source_id)} |
| 123 |
|
| 124 |
@classmethod |
| 125 |
def get_instance(cls, file_descriptor: Union["FileDescriptor", str]) -> "FileDescriptor": |
| 126 |
if not isinstance(file_descriptor, cls): |
| 127 |
return cls(file_descriptor) |
| 128 |
return file_descriptor |
| 129 |
|
| 130 |
|
| 131 |
class BlobDescriptor(DescriptorBase): |
| 132 |
"""BlobDescriptor is used to describe a blob that will be used as an ingestion source""" |
| 133 |
|
| 134 |
_BLOB_URI = "blob_uri" |
| 135 |
|
| 136 |
def __init__(self, path: str, size: Optional[int] = None, source_id: OptionalUUID = None): |
| 137 |
""" |
| 138 |
:param path: blob uri. |
| 139 |
:type path: str. |
| 140 |
:param size: estimated size of file if known. |
| 141 |
:type size: Optional[int]. |
| 142 |
:param source_id: a v4 uuid to serve as the source's id. |
| 143 |
:type source_id: OptionalUUID |
| 144 |
""" |
| 145 |
self.path: str = path |
| 146 |
self.size: Optional[int] = size |
| 147 |
self.source_id: uuid.UUID = ensure_uuid(source_id) |
| 148 |
|
| 149 |
def get_tracing_attributes(self) -> dict: |
| 150 |
# Remove query parameters from self.path, if exists |
| 151 |
if self.path: |
| 152 |
obfuscated_path = self.path.split("?")[0].split(";")[0] |
| 153 |
return {self._BLOB_URI: obfuscated_path, self._SOURCE_ID: str(self.source_id)} |
| 154 |
|
| 155 |
def fill_size(self): |
| 156 |
if not self.size: |
| 157 |
self.size = BlobClient.from_blob_url(self.path).get_blob_properties().size |
| 158 |
|
| 159 |
|
| 160 |
class StreamDescriptor(DescriptorBase): |
| 161 |
"""StreamDescriptor is used to describe a stream that will be used as ingestion source""" |
| 162 |
|
| 163 |
_STREAM_NAME = "stream_name" |
| 164 |
|
| 165 |
# TODO: currently we always assume that streams are gz compressed (will get compressed before sending), should we expand that? |
| 166 |
def __init__( |
| 167 |
self, stream: IO[AnyStr], source_id: OptionalUUID = None, is_compressed: bool = False, stream_name: Optional[str] = None, size: Optional[int] = None |
| 168 |
): |
| 169 |
""" |
| 170 |
:param stream: in-memory stream object. |
| 171 |
:type stream: io.BaseIO |
| 172 |
:param source_id: a v4 uuid to serve as the sources id. |
| 173 |
:type source_id: OptionalUUID |
| 174 |
:param is_compressed: specify if the provided stream is compressed |
| 175 |
:type is_compressed: boolean |
| 176 |
""" |
| 177 |
self.stream: IO[AnyStr] = stream |
| 178 |
self.source_id: uuid.UUID = ensure_uuid(source_id) |
| 179 |
self.is_compressed: bool = is_compressed |
| 180 |
self.stream_name: str = stream_name |
| 181 |
if self.stream_name is None: |
| 182 |
self.stream_name = "stream" |
| 183 |
if is_compressed: |
| 184 |
self.stream_name += ".gz" |
| 185 |
self.size: Optional[int] = size |
| 186 |
|
| 187 |
def compress_stream(self) -> None: |
| 188 |
stream = self.stream |
| 189 |
zipped_stream = BytesIO() |
| 190 |
stream_buffer = stream.read() |
| 191 |
with GzipFile(filename="data", fileobj=zipped_stream, mode="wb") as f_out: |
| 192 |
if isinstance(stream_buffer, str): |
| 193 |
data = bytes(stream_buffer, "utf-8") |
| 194 |
f_out.write(data) |
| 195 |
else: |
| 196 |
f_out.write(stream_buffer) |
| 197 |
zipped_stream.seek(0) |
| 198 |
self.is_compressed = True |
| 199 |
self.stream_name += ".gz" |
| 200 |
self.stream = zipped_stream |
| 201 |
|
| 202 |
@staticmethod |
| 203 |
def from_file_descriptor(file_descriptor: Union[FileDescriptor, str]) -> "StreamDescriptor": |
| 204 |
""" |
| 205 |
Transforms FileDescriptor instance into StreamDescriptor instance. Note that stream is open when instance is returned |
| 206 |
:param Union[FileDescriptor, str] file_descriptor: File Descriptor instance |
| 207 |
:return new StreamDescriptor instance |
| 208 |
""" |
| 209 |
descriptor = FileDescriptor.get_instance(file_descriptor) |
| 210 |
stream = open(descriptor.path, "rb") |
| 211 |
is_compressed = descriptor.path.endswith(".gz") or descriptor.path.endswith(".zip") |
| 212 |
stream_descriptor = StreamDescriptor(stream, descriptor.source_id, is_compressed, descriptor.stream_name, descriptor.size) |
| 213 |
return stream_descriptor |
| 214 |
|
| 215 |
@classmethod |
| 216 |
def get_instance(cls, stream_descriptor: Union["StreamDescriptor", IO[AnyStr]]) -> "StreamDescriptor": |
| 217 |
if not isinstance(stream_descriptor, cls): |
| 218 |
descriptor = cls(stream_descriptor) |
| 219 |
else: |
| 220 |
descriptor = copy(stream_descriptor) |
| 221 |
return descriptor |
| 222 |
|
| 223 |
def get_tracing_attributes(self) -> dict: |
| 224 |
return {self._STREAM_NAME: self.stream_name, self._SOURCE_ID: str(self.source_id)} |