Debian Codegraph @

azure-kusto-ingest/tests/test_e2e_ingest.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import asyncio
4 import io
5 import os
6 import pathlib
7 import platform
8 import pytest
9 import random
10 import sys
11 import time
12 import uuid
13 from azure.identity import DefaultAzureCredential
14 from typing import Optional, ClassVar, Callable
15
16 from azure.kusto.data.env_utils import get_env, prepare_app_key_auth
17 from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
18 from azure.kusto.data.aio import KustoClient as AsyncKustoClient
19 from azure.kusto.data.data_format import DataFormat, IngestionMappingKind
20 from azure.kusto.data.exceptions import KustoServiceError
21 from azure.kusto.ingest import (
22 QueuedIngestClient,
23 KustoStreamingIngestClient,
24 IngestionProperties,
25 ColumnMapping,
26 ValidationPolicy,
27 ValidationOptions,
28 ValidationImplications,
29 ReportLevel,
30 ReportMethod,
31 FileDescriptor,
32 BlobDescriptor,
33 StreamDescriptor,
34 ManagedStreamingIngestClient,
35 )
36 from azure.kusto.ingest._resource_manager import _ResourceUri
37
38
39 @pytest.fixture(params=["ManagedStreaming", "NormalClient"])
40 def is_managed_streaming(request):
41 return request.param == "ManagedStreaming"
42
43
44 @pytest.fixture(scope="module")
45 def event_loop_policy(request):
46 if platform.system() == "Windows":
47 return asyncio.WindowsSelectorEventLoopPolicy()
48 return asyncio.DefaultEventLoopPolicy()
49
50
51 class TestE2E:
52 """A class to define mappings to deft table."""
53
54 input_folder_path: ClassVar[str]
55 engine_cs: ClassVar[Optional[str]]
56 dm_cs: ClassVar[Optional[str]]
57 test_db: ClassVar[Optional[str]]
58 client: ClassVar[KustoClient]
59 test_table: ClassVar[str]
60 current_count: ClassVar[int]
61 ingest_client: ClassVar[QueuedIngestClient]
62 streaming_ingest_client: ClassVar[KustoStreamingIngestClient]
63 managed_streaming_ingest_client: ClassVar[ManagedStreamingIngestClient]
64 csv_file_path: ClassVar[str]
65 tsv_file_path: ClassVar[str]
66 zipped_csv_file_path: ClassVar[str]
67 json_file_path: ClassVar[str]
68 zipped_json_file_path: ClassVar[str]
69 cred: ClassVar[Callable[[], DefaultAzureCredential]]
70 async_cred: ClassVar[Callable[[], DefaultAzureCredential]]
71 dm_kcsb: KustoConnectionStringBuilder
72
73 CHUNK_SIZE = 1024
74
75 @staticmethod
76 def get_test_table_csv_mappings():
77 """A method to define csv mappings to test table."""
78 mappings = list()
79 mappings.append(ColumnMapping(column_name="rownumber", column_type="int", ordinal=0))
80 mappings.append(ColumnMapping(column_name="rowguid", column_type="string", ordinal=1))
81 mappings.append(ColumnMapping(column_name="xdouble", column_type="real", ordinal=2))
82 mappings.append(ColumnMapping(column_name="xfloat", column_type="real", ordinal=3))
83 mappings.append(ColumnMapping(column_name="xbool", column_type="bool", ordinal=4))
84 mappings.append(ColumnMapping(column_name="xint16", column_type="int", ordinal=5))
85 mappings.append(ColumnMapping(column_name="xint32", column_type="int", ordinal=6))
86 mappings.append(ColumnMapping(column_name="xint64", column_type="long", ordinal=7))
87 mappings.append(ColumnMapping(column_name="xuint8", column_type="long", ordinal=8))
88 mappings.append(ColumnMapping(column_name="xuint16", column_type="long", ordinal=9))
89 mappings.append(ColumnMapping(column_name="xuint32", column_type="long", ordinal=10))
90 mappings.append(ColumnMapping(column_name="xuint64", column_type="long", ordinal=11))
91 mappings.append(ColumnMapping(column_name="xdate", column_type="datetime", ordinal=12))
92 mappings.append(ColumnMapping(column_name="xsmalltext", column_type="string", ordinal=13))
93 mappings.append(ColumnMapping(column_name="xtext", column_type="string", ordinal=14))
94 mappings.append(ColumnMapping(column_name="xnumberAsText", column_type="string", ordinal=15))
95 mappings.append(ColumnMapping(column_name="xtime", column_type="timespan", ordinal=16))
96 mappings.append(ColumnMapping(column_name="xtextWithNulls", column_type="string", ordinal=17))
97 mappings.append(ColumnMapping(column_name="xdynamicWithNulls", column_type="dynamic", ordinal=18))
98 return mappings
99
100 @staticmethod
101 def table_json_mappings():
102 """A method to define json mappings to test table."""
103 mappings = list()
104 mappings.append(ColumnMapping(column_name="rownumber", path="$.rownumber", column_type="int"))
105 mappings.append(ColumnMapping(column_name="rowguid", path="$.rowguid", column_type="string"))
106 mappings.append(ColumnMapping(column_name="xdouble", path="$.xdouble", column_type="real"))
107 mappings.append(ColumnMapping(column_name="xfloat", path="$.xfloat", column_type="real"))
108 mappings.append(ColumnMapping(column_name="xbool", path="$.xbool", column_type="bool"))
109 mappings.append(ColumnMapping(column_name="xint16", path="$.xint16", column_type="int"))
110 mappings.append(ColumnMapping(column_name="xint32", path="$.xint32", column_type="int"))
111 mappings.append(ColumnMapping(column_name="xint64", path="$.xint64", column_type="long"))
112 mappings.append(ColumnMapping(column_name="xuint8", path="$.xuint8", column_type="long"))
113 mappings.append(ColumnMapping(column_name="xuint16", path="$.xuint16", column_type="long"))
114 mappings.append(ColumnMapping(column_name="xuint32", path="$.xuint32", column_type="long"))
115 mappings.append(ColumnMapping(column_name="xuint64", path="$.xuint64", column_type="long"))
116 mappings.append(ColumnMapping(column_name="xdate", path="$.xdate", column_type="datetime"))
117 mappings.append(ColumnMapping(column_name="xsmalltext", path="$.xsmalltext", column_type="string"))
118 mappings.append(ColumnMapping(column_name="xtext", path="$.xtext", column_type="string"))
119 mappings.append(ColumnMapping(column_name="xnumberAsText", path="$.xnumberAsText", column_type="string"))
120 mappings.append(ColumnMapping(column_name="xtime", path="$.xtime", column_type="timespan"))
121 mappings.append(ColumnMapping(column_name="xtextWithNulls", path="$.xtextWithNulls", column_type="string"))
122 mappings.append(ColumnMapping(column_name="xdynamicWithNulls", path="$.xdynamicWithNulls", column_type="dynamic"))
123 return mappings
124
125 @staticmethod
126 def table_json_mapping_reference():
127 """A method to get json mappings reference to test table."""
128 return """'['
129 ' { "column" : "rownumber", "datatype" : "int", "Properties":{"Path":"$.rownumber"}},'
130 ' { "column" : "rowguid", "datatype" : "string", "Properties":{"Path":"$.rowguid"}},'
131 ' { "column" : "xdouble", "datatype" : "real", "Properties":{"Path":"$.xdouble"}},'
132 ' { "column" : "xfloat", "datatype" : "real", "Properties":{"Path":"$.xfloat"}},'
133 ' { "column" : "xbool", "datatype" : "bool", "Properties":{"Path":"$.xbool"}},'
134 ' { "column" : "xint16", "datatype" : "int", "Properties":{"Path":"$.xint16"}},'
135 ' { "column" : "xint32", "datatype" : "int", "Properties":{"Path":"$.xint32"}},'
136 ' { "column" : "xint64", "datatype" : "long", "Properties":{"Path":"$.xint64"}},'
137 ' { "column" : "xuint8", "datatype" : "long", "Properties":{"Path":"$.xuint8"}},'
138 ' { "column" : "xuint16", "datatype" : "long", "Properties":{"Path":"$.xuint16"}},'
139 ' { "column" : "xuint32", "datatype" : "long", "Properties":{"Path":"$.xuint32"}},'
140 ' { "column" : "xuint64", "datatype" : "long", "Properties":{"Path":"$.xuint64"}},'
141 ' { "column" : "xdate", "datatype" : "datetime", "Properties":{"Path":"$.xdate"}},'
142 ' { "column" : "xsmalltext", "datatype" : "string", "Properties":{"Path":"$.xsmalltext"}},'
143 ' { "column" : "xtext", "datatype" : "string", "Properties":{"Path":"$.xtext"}},'
144 ' { "column" : "xnumberAsText", "datatype" : "string", "Properties":{"Path":"$.rowguid"}},'
145 ' { "column" : "xtime", "datatype" : "timespan", "Properties":{"Path":"$.xtime"}},'
146 ' { "column" : "xtextWithNulls", "datatype" : "string", "Properties":{"Path":"$.xtextWithNulls"}},'
147 ' { "column" : "xdynamicWithNulls", "datatype" : "dynamic", "Properties":{"Path":"$.xdynamicWithNulls"}},'
148 ']'"""
149
150 @staticmethod
151 def get_file_path() -> str:
152 current_dir = os.getcwd()
153 path_parts = ["azure-kusto-ingest", "tests", "input"]
154 missing_path_parts = []
155 for path_part in path_parts:
156 if path_part not in current_dir:
157 missing_path_parts.append(path_part)
158 return os.path.join(current_dir, *missing_path_parts)
159
160 @classmethod
161 def engine_kcsb_from_env(cls, is_async=False) -> KustoConnectionStringBuilder:
162 return KustoConnectionStringBuilder.with_azure_token_credential(
163 cls.engine_cs,
164 credential=cls.cred() if not is_async else cls.async_cred(),
165 )
166
167 @classmethod
168 def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder:
169 return KustoConnectionStringBuilder.with_azure_token_credential(cls.dm_cs, credential=cls.cred())
170
171 @classmethod
172 def setup_class(cls):
173 # DM CS can be composed from engine CS
174 cls.engine_cs = get_env("ENGINE_CONNECTION_STRING")
175 cls.dm_cs = get_env("DM_CONNECTION_STRING", default=cls.engine_cs.replace("//", "//ingest-"))
176
177 # Called to set the env variables for the default azure credentials
178 prepare_app_key_auth(optional=True)
179
180 cls.test_db = get_env("TEST_DATABASE")
181 cls.test_blob = get_env("TEST_BLOB", optional=True)
182
183 cls.cred = lambda: DefaultAzureCredential(exclude_interactive_browser_credential=False)
184 # Async credentials don't support interactive browser authentication for now, so until they do, we'll use the sync default credential for async tests
185 cls.async_cred = lambda: DefaultAzureCredential(exclude_interactive_browser_credential=False)
186
187 # Init clients
188 python_version = "_".join([str(v) for v in sys.version_info[:3]])
189 cls.test_table = "python_test_{0}_{1}_{2}".format(python_version, str(int(time.time())), random.randint(1, 100000))
190
191 cls.client = KustoClient(cls.engine_kcsb_from_env())
192 cls.ingest_client = QueuedIngestClient(cls.dm_kcsb_from_env())
193 cls.streaming_ingest_client = KustoStreamingIngestClient(cls.engine_kcsb_from_env())
194 cls.managed_streaming_ingest_client = ManagedStreamingIngestClient(cls.engine_kcsb_from_env(), cls.dm_kcsb_from_env())
195
196 cls.input_folder_path = cls.get_file_path()
197
198 cls.csv_file_path = os.path.join(cls.input_folder_path, "dataset.csv")
199 cls.tsv_file_path = os.path.join(cls.input_folder_path, "dataset.tsv")
200 cls.zipped_csv_file_path = os.path.join(cls.input_folder_path, "dataset.csv.gz")
201 cls.json_file_path = os.path.join(cls.input_folder_path, "dataset.json")
202 cls.zipped_json_file_path = os.path.join(cls.input_folder_path, "dataset.jsonz.gz")
203
204 cls.current_count = 0
205
206 cls.client.execute(
207 cls.test_db,
208 f".create table {cls.test_table} (rownumber: int, rowguid: string, xdouble: real, xfloat: real, xbool: bool, xint16: int, xint32: int, xint64: long, xuint8: long, xuint16: long, xuint32: long, xuint64: long, xdate: datetime, xsmalltext: string, xtext: string, xnumberAsText: string, xtime: timespan, xtextWithNulls: string, xdynamicWithNulls: dynamic)",
209 )
210 cls.client.execute(cls.test_db, f".create table {cls.test_table} ingestion json mapping 'JsonMapping' {cls.table_json_mapping_reference()}")
211
212 cls.client.execute(cls.test_db, f".alter table {cls.test_table} policy streamingingestion enable ")
213
214 # Clear the cache to guarantee that subsequent streaming ingestion requests incorporate database and table schema changes
215 # See https://docs.microsoft.com/azure/data-explorer/kusto/management/data-ingestion/clear-schema-cache-command
216 cls.client.execute(cls.test_db, ".clear database cache streamingingestion schema")
217
218 @classmethod
219 def teardown_class(cls):
220 cls.client.execute(cls.test_db, ".drop table {} ifexists".format(cls.test_table))
221 cls.client.close()
222 cls.ingest_client.close()
223 cls.streaming_ingest_client.close()
224 cls.managed_streaming_ingest_client.close()
225
226 @classmethod
227 async def get_async_client(cls) -> AsyncKustoClient:
228 return AsyncKustoClient(cls.engine_kcsb_from_env(is_async=True))
229
230 # assertions
231 @classmethod
232 async def assert_rows_added(cls, expected: int, timeout=60):
233 actual = 0
234 while timeout > 0:
235 time.sleep(1)
236 timeout -= 1
237
238 try:
239 command = "{} | count".format(cls.test_table)
240 response = cls.client.execute(cls.test_db, command)
241 async_client = await cls.get_async_client()
242 response_from_async = await async_client.execute(cls.test_db, command)
243 except KustoServiceError:
244 continue
245
246 if response is not None:
247 row = response.primary_results[0][0]
248 row_async = response_from_async.primary_results[0][0]
249 actual = int(row["Count"]) - cls.current_count
250 # this is done to allow for data to arrive properly
251 if actual >= expected:
252 assert row_async == row, "Mismatch answers between async('{0}') and sync('{1}') clients".format(row_async, row)
253 break
254
255 cls.current_count += actual
256 assert actual == expected, "Row count expected = {0}, while actual row count = {1}".format(expected, actual)
257
258 @pytest.mark.asyncio
259 async def test_csv_ingest_existing_table(self, is_managed_streaming):
260 csv_ingest_props = IngestionProperties(
261 self.test_db,
262 self.test_table,
263 data_format=DataFormat.CSV,
264 column_mappings=self.get_test_table_csv_mappings(),
265 report_level=ReportLevel.FailuresAndSuccesses,
266 flush_immediately=True,
267 )
268
269 client = self.streaming_ingest_client if is_managed_streaming else self.ingest_client
270
271 for f in [self.csv_file_path, self.zipped_csv_file_path]:
272 client.ingest_from_file(f, csv_ingest_props)
273
274 await self.assert_rows_added(20)
275
276 @pytest.mark.asyncio
277 async def test_csv_ingest_ignore_first_record(self, is_managed_streaming):
278 csv_ingest_props = IngestionProperties(
279 self.test_db,
280 self.test_table,
281 data_format=DataFormat.CSV,
282 column_mappings=self.get_test_table_csv_mappings(),
283 report_level=ReportLevel.FailuresAndSuccesses,
284 flush_immediately=True,
285 ignore_first_record=True,
286 )
287
288 for f in [self.csv_file_path, self.zipped_csv_file_path]:
289 self.ingest_client.ingest_from_file(f, csv_ingest_props)
290
291 await self.assert_rows_added(18)
292
293 @pytest.mark.asyncio
294 async def test_json_ingest_existing_table(self):
295 json_ingestion_props = IngestionProperties(
296 self.test_db,
297 self.test_table,
298 flush_immediately=True,
299 data_format=DataFormat.JSON,
300 column_mappings=self.table_json_mappings(),
301 report_level=ReportLevel.FailuresAndSuccesses,
302 )
303
304 for f in [self.json_file_path, self.zipped_json_file_path]:
305 self.ingest_client.ingest_from_file(f, json_ingestion_props)
306
307 await self.assert_rows_added(4)
308
309 @pytest.mark.asyncio
310 async def test_json_ingest_existing_table_no_mapping(self):
311 json_ingestion_props = IngestionProperties(
312 self.test_db,
313 self.test_table,
314 flush_immediately=True,
315 data_format=DataFormat.JSON,
316 report_level=ReportLevel.FailuresAndSuccesses,
317 )
318
319 for f in [self.json_file_path, self.zipped_json_file_path]:
320 self.ingest_client.ingest_from_file(f, json_ingestion_props)
321
322 await self.assert_rows_added(4)
323
324 @pytest.mark.asyncio
325 async def test_ingest_complicated_props(self):
326 validation_policy = ValidationPolicy(
327 validation_options=ValidationOptions.ValidateCsvInputConstantColumns, validation_implications=ValidationImplications.Fail
328 )
329 json_ingestion_props = IngestionProperties(
330 self.test_db,
331 self.test_table,
332 data_format=DataFormat.JSON,
333 column_mappings=self.table_json_mappings(),
334 additional_tags=["a", "b"],
335 ingest_if_not_exists=["aaaa", "bbbb"],
336 ingest_by_tags=["ingestByTag"],
337 drop_by_tags=["drop", "drop-by"],
338 flush_immediately=False,
339 report_level=ReportLevel.FailuresAndSuccesses,
340 report_method=ReportMethod.Queue,
341 validation_policy=validation_policy,
342 )
343
344 file_paths = [self.json_file_path, self.zipped_json_file_path]
345 fds = [FileDescriptor(fp, 0, uuid.uuid4()) for fp in file_paths]
346
347 for fd in fds:
348 self.ingest_client.ingest_from_file(fd, json_ingestion_props)
349
350 await self.assert_rows_added(4)
351
352 @pytest.mark.asyncio
353 async def test_ingest_from_stream(self, is_managed_streaming):
354 validation_policy = ValidationPolicy(
355 validation_options=ValidationOptions.ValidateCsvInputConstantColumns, validation_implications=ValidationImplications.Fail
356 )
357 json_ingestion_props = IngestionProperties(
358 self.test_db,
359 self.test_table,
360 data_format=DataFormat.JSON,
361 column_mappings=self.table_json_mappings(),
362 additional_tags=["a", "b"],
363 ingest_if_not_exists=["aaaa", "bbbb"],
364 ingest_by_tags=["ingestByTag"],
365 drop_by_tags=["drop", "drop-by"],
366 flush_immediately=False,
367 report_level=ReportLevel.FailuresAndSuccesses,
368 report_method=ReportMethod.Queue,
369 validation_policy=validation_policy,
370 )
371 text = io.StringIO(pathlib.Path(self.json_file_path).read_text())
372 zipped = io.BytesIO(pathlib.Path(self.zipped_json_file_path).read_bytes())
373
374 client = self.managed_streaming_ingest_client if is_managed_streaming else self.ingest_client
375
376 client.ingest_from_stream(text, json_ingestion_props)
377 client.ingest_from_stream(StreamDescriptor(zipped, is_compressed=True), json_ingestion_props)
378
379 await self.assert_rows_added(4)
380
381 @pytest.mark.asyncio
382 async def test_json_ingestion_ingest_by_tag(self):
383 json_ingestion_props = IngestionProperties(
384 self.test_db,
385 self.test_table,
386 data_format=DataFormat.JSON,
387 column_mappings=self.table_json_mappings(),
388 ingest_if_not_exists=["ingestByTag"],
389 report_level=ReportLevel.FailuresAndSuccesses,
390 drop_by_tags=["drop", "drop-by"],
391 flush_immediately=True,
392 )
393
394 for f in [self.json_file_path, self.zipped_json_file_path]:
395 self.ingest_client.ingest_from_file(f, json_ingestion_props)
396
397 await self.assert_rows_added(0)
398
399 @pytest.mark.asyncio
400 async def test_tsv_ingestion_csv_mapping(self):
401 tsv_ingestion_props = IngestionProperties(
402 self.test_db,
403 self.test_table,
404 flush_immediately=True,
405 data_format=DataFormat.TSV,
406 column_mappings=self.get_test_table_csv_mappings(),
407 report_level=ReportLevel.FailuresAndSuccesses,
408 )
409
410 self.ingest_client.ingest_from_file(self.tsv_file_path, tsv_ingestion_props)
411
412 await self.assert_rows_added(10)
413
414 @pytest.mark.asyncio
415 async def test_ingest_blob(self):
416 if not self.test_blob:
417 pytest.skip("Provide blob SAS uri with 'dataset.csv'")
418
419 csv_ingest_props = IngestionProperties(
420 self.test_db,
421 self.test_table,
422 data_format=DataFormat.CSV,
423 column_mappings=self.get_test_table_csv_mappings(),
424 report_level=ReportLevel.FailuresAndSuccesses,
425 flush_immediately=True,
426 )
427
428 blob_len = 1578
429 self.ingest_client.ingest_from_blob(BlobDescriptor(self.test_blob, blob_len), csv_ingest_props)
430
431 await self.assert_rows_added(10)
432
433 # Don't provide size hint
434 self.ingest_client.ingest_from_blob(BlobDescriptor(self.test_blob, size=None), csv_ingest_props)
435
436 await self.assert_rows_added(10)
437
438 @pytest.mark.asyncio
439 async def test_streaming_ingest_from_opened_file(self, is_managed_streaming):
440 ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, data_format=DataFormat.CSV)
441
442 client = self.managed_streaming_ingest_client if is_managed_streaming else self.streaming_ingest_client
443 with open(self.csv_file_path, "r") as stream:
444 client.ingest_from_stream(stream, ingestion_properties=ingestion_properties)
445
446 await self.assert_rows_added(10, timeout=120)
447
448 @pytest.mark.asyncio
449 async def test_streaming_ingest_from_csv_file(self):
450 ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, flush_immediately=True, data_format=DataFormat.CSV)
451
452 for f in [self.csv_file_path, self.zipped_csv_file_path]:
453 self.streaming_ingest_client.ingest_from_file(f, ingestion_properties=ingestion_properties)
454
455 await self.assert_rows_added(20, timeout=120)
456
457 @pytest.mark.asyncio
458 async def test_streaming_ingest_from_json_file(self):
459 ingestion_properties = IngestionProperties(
460 database=self.test_db,
461 table=self.test_table,
462 flush_immediately=True,
463 data_format=DataFormat.JSON,
464 ingestion_mapping_reference="JsonMapping",
465 ingestion_mapping_kind=IngestionMappingKind.JSON,
466 )
467
468 for f in [self.json_file_path, self.zipped_json_file_path]:
469 self.streaming_ingest_client.ingest_from_file(f, ingestion_properties=ingestion_properties)
470
471 await self.assert_rows_added(4, timeout=120)
472
473 @pytest.mark.asyncio
474 async def test_streaming_ingest_from_csv_io_streams(self):
475 ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, data_format=DataFormat.CSV)
476 byte_sequence = b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
477 bytes_stream = io.BytesIO(byte_sequence)
478 self.streaming_ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
479
480 str_sequence = '0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null'
481 str_stream = io.StringIO(str_sequence)
482 self.streaming_ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
483
484 await self.assert_rows_added(2, timeout=120)
485
486 @pytest.mark.asyncio
487 async def test_streaming_ingest_from_json_io_streams(self):
488 ingestion_properties = IngestionProperties(
489 database=self.test_db,
490 table=self.test_table,
491 data_format=DataFormat.JSON,
492 flush_immediately=True,
493 ingestion_mapping_reference="JsonMapping",
494 ingestion_mapping_kind=IngestionMappingKind.JSON,
495 )
496
497 byte_sequence = b'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}'
498 bytes_stream = io.BytesIO(byte_sequence)
499 self.streaming_ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
500
501 str_sequence = '{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}'
502 str_stream = io.StringIO(str_sequence)
503 self.streaming_ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
504
505 await self.assert_rows_added(2, timeout=120)
506
507 @pytest.mark.asyncio
508 async def test_streaming_ingest_from_dataframe(self):
509 from pandas import DataFrame
510
511 fields = [
512 "rownumber",
513 "rowguid",
514 "xdouble",
515 "xfloat",
516 "xbool",
517 "xint16",
518 "xint32",
519 "xint64",
520 "xunit8",
521 "xuint16",
522 "xunit32",
523 "xunit64",
524 "xdate",
525 "xsmalltext",
526 "xtext",
527 "xnumberAsText",
528 "xtime",
529 "xtextWithNulls",
530 "xdynamicWithNulls",
531 ]
532
533 guid = uuid.uuid4()
534
535 dynamic_value = ["[email protected]", "[email protected]", "[email protected]"]
536 rows = [[0, str(guid), 0.0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, "2014-01-01T01:01:01Z", "Zero", "Zero", "0", "00:00:00", None, dynamic_value]]
537 df = DataFrame(data=rows, columns=fields)
538 ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, flush_immediately=True)
539 self.ingest_client.ingest_from_dataframe(df, ingestion_properties)
540
541 await self.assert_rows_added(1, timeout=120)
542
543 a = self.client.execute(self.test_db, f"{self.test_table} | where rowguid == '{guid}'")
544 assert a.primary_results[0].rows[0]["xdynamicWithNulls"] == dynamic_value
545
546 @pytest.mark.asyncio
547 async def test_streaming_ingest_from_blob(self, is_managed_streaming):
548 ingestion_properties = IngestionProperties(
549 database=self.test_db,
550 table=self.test_table,
551 data_format=DataFormat.JSON,
552 ingestion_mapping_reference="JsonMapping",
553 ingestion_mapping_kind=IngestionMappingKind.JSON,
554 )
555 export_containers_list = self.ingest_client._resource_manager._kusto_client.execute("NetDefaultDB", ".show export containers")
556 containers = [_ResourceUri(s["StorageRoot"]) for s in export_containers_list.primary_results[0]]
557
558 for c in containers:
559 self.ingest_client._resource_manager._ranked_storage_account_set.add_storage_account(c.storage_account_name)
560
561 with FileDescriptor(self.json_file_path).open(False) as stream:
562 blob_descriptor = self.ingest_client.upload_blob(
563 containers,
564 FileDescriptor(self.json_file_path),
565 ingestion_properties.database,
566 ingestion_properties.table,
567 stream,
568 None,
569 10 * 60,
570 3,
571 )
572 if is_managed_streaming:
573 self.managed_streaming_ingest_client.ingest_from_blob(blob_descriptor, ingestion_properties)
574 else:
575 self.streaming_ingest_client.ingest_from_blob(blob_descriptor, ingestion_properties)
576
577 await self.assert_rows_added(2, timeout=120)