Debian Codegraph @

azure-kusto-ingest/tests/test_descriptors.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import sys
4 import uuid
5 from io import BytesIO
6 from os import path
7
8 import pytest
9
10 from azure.kusto.ingest import FileDescriptor, BlobDescriptor, StreamDescriptor
11
12
13 class TestDescriptors:
14 """Test class for FileDescriptor and BlobDescriptor."""
15
16 # this is the size with LF line endings
17 uncompressed_size = 1569
18 # this is the size with CRLF line endings
19 uncompressed_size_2 = 1578
20 mock_size = 10
21
22 INVALID_UUID = "12345"
23 TEST_UUID_STR = "5bcc12b7-e35c-4c76-a40a-2d89e6c2c7dd"
24 TEST_UUID = uuid.UUID("5bcc12b7-e35c-4c76-a40a-2d89e6c2c7dd", version=4)
25
26 def test_unzipped_file_with_size(self):
27 """Tests FileDescriptor with size and unzipped file."""
28 filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv")
29 descriptor = FileDescriptor(filePath, self.mock_size)
30 with descriptor.open(True) as stream:
31 assert descriptor.size == self.mock_size
32 assert descriptor.stream_name.endswith(".csv.gz")
33 if sys.version_info[0] >= 3:
34 assert stream.readable()
35 assert stream.tell() == 0
36
37 assert stream.closed is True
38
39 def test_unzipped_file_without_size(self):
40 """Tests FileDescriptor without size and unzipped file."""
41 filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv")
42 descriptor = FileDescriptor(filePath, 0)
43 with descriptor.open(True) as stream:
44 # TODO: since we don't know if the file is opened on CRLF system or an LF system, allow both sizes
45 # a more robust approach would be to open the file and check
46 assert descriptor.size in (self.uncompressed_size, self.uncompressed_size_2)
47 assert descriptor.stream_name.endswith(".csv.gz")
48 if sys.version_info[0] >= 3:
49 assert stream.readable()
50 assert stream.tell() == 0
51
52 assert stream.closed is True
53
54 def test_zipped_file_with_size(self):
55 """Tests FileDescriptor with size and zipped file."""
56 filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv.gz")
57 descriptor = FileDescriptor(filePath, self.mock_size)
58 with descriptor.open(False) as stream:
59 assert descriptor.size == self.mock_size
60 assert descriptor.stream_name.endswith(".csv.gz")
61 if sys.version_info[0] >= 3:
62 assert stream.readable()
63 assert stream.tell() == 0
64
65 assert stream.closed is True
66
67 def test_gzip_file_without_size(self):
68 """Tests FileDescriptor without size and zipped file."""
69 filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv.gz")
70 descriptor = FileDescriptor(filePath, 0)
71 with descriptor.open(False) as stream:
72 assert descriptor.size == self.uncompressed_size
73 assert descriptor.stream_name.endswith(".csv.gz")
74 if sys.version_info[0] >= 3:
75 assert stream.readable()
76 assert stream.tell() == 0
77
78 assert stream.closed is True
79
80 def test_zip_file_without_size(self):
81 """Tests FileDescriptor without size and zipped file."""
82 filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv.zip")
83 descriptor = FileDescriptor(filePath, 0)
84 with descriptor.open(False) as stream:
85 # the zip archive contains 2 copies of the source file
86 assert descriptor.size == self.uncompressed_size * 2
87 assert descriptor.stream_name.endswith(".csv.zip")
88 if sys.version_info[0] >= 3:
89 assert stream.readable()
90 assert stream.tell() == 0
91
92 assert stream.closed is True
93
94 def test_unzipped_file_dont_compress(self):
95 """Tests FileDescriptor with size and unzipped file."""
96 filePath = path.join(path.dirname(path.abspath(__file__)), "input", "dataset.csv")
97 descriptor = FileDescriptor(filePath, self.mock_size)
98 with descriptor.open(False) as stream:
99 assert descriptor.size == self.mock_size
100 assert descriptor.stream_name.endswith(".csv")
101 if sys.version_info[0] >= 3:
102 assert stream.readable()
103 assert stream.tell() == 0
104
105 assert stream.closed is True
106
107 def test_uuid_stream_descriptor(self):
108 dummy_stream = BytesIO(b"dummy")
109
110 descriptor = StreamDescriptor(dummy_stream)
111 assert descriptor.source_id
112 assert descriptor.source_id != TestDescriptors.TEST_UUID
113 assert uuid.UUID(str(descriptor.source_id), version=4)
114
115 descriptor = StreamDescriptor(dummy_stream, source_id=TestDescriptors.TEST_UUID_STR)
116 assert descriptor.source_id == TestDescriptors.TEST_UUID
117
118 descriptor = StreamDescriptor(dummy_stream, source_id=TestDescriptors.TEST_UUID)
119 assert descriptor.source_id == TestDescriptors.TEST_UUID
120
121 with pytest.raises(ValueError):
122 StreamDescriptor(dummy_stream, source_id=TestDescriptors.INVALID_UUID)
123
124 def test_uuid_file_descriptor(self):
125 dummy_file = "dummy"
126
127 descriptor = FileDescriptor(dummy_file)
128 assert descriptor.source_id
129 assert descriptor.source_id != TestDescriptors.TEST_UUID
130 assert uuid.UUID(str(descriptor.source_id), version=4)
131
132 descriptor = FileDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID_STR)
133 assert descriptor.source_id == TestDescriptors.TEST_UUID
134
135 descriptor = FileDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID)
136 assert descriptor.source_id == TestDescriptors.TEST_UUID
137
138 with pytest.raises(ValueError):
139 FileDescriptor(dummy_file, source_id=TestDescriptors.INVALID_UUID)
140
141 def test_uuid_blob_descriptor(self):
142 dummy_file = "dummy"
143
144 descriptor = BlobDescriptor(dummy_file)
145 assert descriptor.source_id
146 assert descriptor.source_id != TestDescriptors.TEST_UUID
147 assert uuid.UUID(str(descriptor.source_id), version=4)
148
149 descriptor = BlobDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID_STR)
150 assert descriptor.source_id == TestDescriptors.TEST_UUID
151
152 descriptor = BlobDescriptor(dummy_file, source_id=TestDescriptors.TEST_UUID)
153 assert descriptor.source_id == TestDescriptors.TEST_UUID
154
155 with pytest.raises(ValueError):
156 BlobDescriptor(dummy_file, source_id=TestDescriptors.INVALID_UUID)