| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
import asyncio |
| 4 |
import io |
| 5 |
import os |
| 6 |
import pathlib |
| 7 |
import platform |
| 8 |
import pytest |
| 9 |
import random |
| 10 |
import sys |
| 11 |
import time |
| 12 |
import uuid |
| 13 |
from azure.identity import DefaultAzureCredential |
| 14 |
from typing import Optional, ClassVar, Callable |
| 15 |
|
| 16 |
from azure.kusto.data.env_utils import get_env, prepare_app_key_auth |
| 17 |
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder |
| 18 |
from azure.kusto.data.aio import KustoClient as AsyncKustoClient |
| 19 |
from azure.kusto.data.data_format import DataFormat, IngestionMappingKind |
| 20 |
from azure.kusto.data.exceptions import KustoServiceError |
| 21 |
from azure.kusto.ingest import ( |
| 22 |
QueuedIngestClient, |
| 23 |
KustoStreamingIngestClient, |
| 24 |
IngestionProperties, |
| 25 |
ColumnMapping, |
| 26 |
ValidationPolicy, |
| 27 |
ValidationOptions, |
| 28 |
ValidationImplications, |
| 29 |
ReportLevel, |
| 30 |
ReportMethod, |
| 31 |
FileDescriptor, |
| 32 |
BlobDescriptor, |
| 33 |
StreamDescriptor, |
| 34 |
ManagedStreamingIngestClient, |
| 35 |
) |
| 36 |
from azure.kusto.ingest._resource_manager import _ResourceUri |
| 37 |
|
| 38 |
|
| 39 |
@pytest.fixture(params=["ManagedStreaming", "NormalClient"]) |
| 40 |
def is_managed_streaming(request): |
| 41 |
return request.param == "ManagedStreaming" |
| 42 |
|
| 43 |
|
| 44 |
@pytest.fixture(scope="module") |
| 45 |
def event_loop_policy(request): |
| 46 |
if platform.system() == "Windows": |
| 47 |
return asyncio.WindowsSelectorEventLoopPolicy() |
| 48 |
return asyncio.DefaultEventLoopPolicy() |
| 49 |
|
| 50 |
|
| 51 |
class TestE2E: |
| 52 |
"""A class to define mappings to deft table.""" |
| 53 |
|
| 54 |
input_folder_path: ClassVar[str] |
| 55 |
engine_cs: ClassVar[Optional[str]] |
| 56 |
dm_cs: ClassVar[Optional[str]] |
| 57 |
test_db: ClassVar[Optional[str]] |
| 58 |
client: ClassVar[KustoClient] |
| 59 |
test_table: ClassVar[str] |
| 60 |
current_count: ClassVar[int] |
| 61 |
ingest_client: ClassVar[QueuedIngestClient] |
| 62 |
streaming_ingest_client: ClassVar[KustoStreamingIngestClient] |
| 63 |
managed_streaming_ingest_client: ClassVar[ManagedStreamingIngestClient] |
| 64 |
csv_file_path: ClassVar[str] |
| 65 |
tsv_file_path: ClassVar[str] |
| 66 |
zipped_csv_file_path: ClassVar[str] |
| 67 |
json_file_path: ClassVar[str] |
| 68 |
zipped_json_file_path: ClassVar[str] |
| 69 |
cred: ClassVar[Callable[[], DefaultAzureCredential]] |
| 70 |
async_cred: ClassVar[Callable[[], DefaultAzureCredential]] |
| 71 |
dm_kcsb: KustoConnectionStringBuilder |
| 72 |
|
| 73 |
CHUNK_SIZE = 1024 |
| 74 |
|
| 75 |
@staticmethod |
| 76 |
def get_test_table_csv_mappings(): |
| 77 |
"""A method to define csv mappings to test table.""" |
| 78 |
mappings = list() |
| 79 |
mappings.append(ColumnMapping(column_name="rownumber", column_type="int", ordinal=0)) |
| 80 |
mappings.append(ColumnMapping(column_name="rowguid", column_type="string", ordinal=1)) |
| 81 |
mappings.append(ColumnMapping(column_name="xdouble", column_type="real", ordinal=2)) |
| 82 |
mappings.append(ColumnMapping(column_name="xfloat", column_type="real", ordinal=3)) |
| 83 |
mappings.append(ColumnMapping(column_name="xbool", column_type="bool", ordinal=4)) |
| 84 |
mappings.append(ColumnMapping(column_name="xint16", column_type="int", ordinal=5)) |
| 85 |
mappings.append(ColumnMapping(column_name="xint32", column_type="int", ordinal=6)) |
| 86 |
mappings.append(ColumnMapping(column_name="xint64", column_type="long", ordinal=7)) |
| 87 |
mappings.append(ColumnMapping(column_name="xuint8", column_type="long", ordinal=8)) |
| 88 |
mappings.append(ColumnMapping(column_name="xuint16", column_type="long", ordinal=9)) |
| 89 |
mappings.append(ColumnMapping(column_name="xuint32", column_type="long", ordinal=10)) |
| 90 |
mappings.append(ColumnMapping(column_name="xuint64", column_type="long", ordinal=11)) |
| 91 |
mappings.append(ColumnMapping(column_name="xdate", column_type="datetime", ordinal=12)) |
| 92 |
mappings.append(ColumnMapping(column_name="xsmalltext", column_type="string", ordinal=13)) |
| 93 |
mappings.append(ColumnMapping(column_name="xtext", column_type="string", ordinal=14)) |
| 94 |
mappings.append(ColumnMapping(column_name="xnumberAsText", column_type="string", ordinal=15)) |
| 95 |
mappings.append(ColumnMapping(column_name="xtime", column_type="timespan", ordinal=16)) |
| 96 |
mappings.append(ColumnMapping(column_name="xtextWithNulls", column_type="string", ordinal=17)) |
| 97 |
mappings.append(ColumnMapping(column_name="xdynamicWithNulls", column_type="dynamic", ordinal=18)) |
| 98 |
return mappings |
| 99 |
|
| 100 |
@staticmethod |
| 101 |
def table_json_mappings(): |
| 102 |
"""A method to define json mappings to test table.""" |
| 103 |
mappings = list() |
| 104 |
mappings.append(ColumnMapping(column_name="rownumber", path="$.rownumber", column_type="int")) |
| 105 |
mappings.append(ColumnMapping(column_name="rowguid", path="$.rowguid", column_type="string")) |
| 106 |
mappings.append(ColumnMapping(column_name="xdouble", path="$.xdouble", column_type="real")) |
| 107 |
mappings.append(ColumnMapping(column_name="xfloat", path="$.xfloat", column_type="real")) |
| 108 |
mappings.append(ColumnMapping(column_name="xbool", path="$.xbool", column_type="bool")) |
| 109 |
mappings.append(ColumnMapping(column_name="xint16", path="$.xint16", column_type="int")) |
| 110 |
mappings.append(ColumnMapping(column_name="xint32", path="$.xint32", column_type="int")) |
| 111 |
mappings.append(ColumnMapping(column_name="xint64", path="$.xint64", column_type="long")) |
| 112 |
mappings.append(ColumnMapping(column_name="xuint8", path="$.xuint8", column_type="long")) |
| 113 |
mappings.append(ColumnMapping(column_name="xuint16", path="$.xuint16", column_type="long")) |
| 114 |
mappings.append(ColumnMapping(column_name="xuint32", path="$.xuint32", column_type="long")) |
| 115 |
mappings.append(ColumnMapping(column_name="xuint64", path="$.xuint64", column_type="long")) |
| 116 |
mappings.append(ColumnMapping(column_name="xdate", path="$.xdate", column_type="datetime")) |
| 117 |
mappings.append(ColumnMapping(column_name="xsmalltext", path="$.xsmalltext", column_type="string")) |
| 118 |
mappings.append(ColumnMapping(column_name="xtext", path="$.xtext", column_type="string")) |
| 119 |
mappings.append(ColumnMapping(column_name="xnumberAsText", path="$.xnumberAsText", column_type="string")) |
| 120 |
mappings.append(ColumnMapping(column_name="xtime", path="$.xtime", column_type="timespan")) |
| 121 |
mappings.append(ColumnMapping(column_name="xtextWithNulls", path="$.xtextWithNulls", column_type="string")) |
| 122 |
mappings.append(ColumnMapping(column_name="xdynamicWithNulls", path="$.xdynamicWithNulls", column_type="dynamic")) |
| 123 |
return mappings |
| 124 |
|
| 125 |
@staticmethod |
| 126 |
def table_json_mapping_reference(): |
| 127 |
"""A method to get json mappings reference to test table.""" |
| 128 |
return """'[' |
| 129 |
' { "column" : "rownumber", "datatype" : "int", "Properties":{"Path":"$.rownumber"}},' |
| 130 |
' { "column" : "rowguid", "datatype" : "string", "Properties":{"Path":"$.rowguid"}},' |
| 131 |
' { "column" : "xdouble", "datatype" : "real", "Properties":{"Path":"$.xdouble"}},' |
| 132 |
' { "column" : "xfloat", "datatype" : "real", "Properties":{"Path":"$.xfloat"}},' |
| 133 |
' { "column" : "xbool", "datatype" : "bool", "Properties":{"Path":"$.xbool"}},' |
| 134 |
' { "column" : "xint16", "datatype" : "int", "Properties":{"Path":"$.xint16"}},' |
| 135 |
' { "column" : "xint32", "datatype" : "int", "Properties":{"Path":"$.xint32"}},' |
| 136 |
' { "column" : "xint64", "datatype" : "long", "Properties":{"Path":"$.xint64"}},' |
| 137 |
' { "column" : "xuint8", "datatype" : "long", "Properties":{"Path":"$.xuint8"}},' |
| 138 |
' { "column" : "xuint16", "datatype" : "long", "Properties":{"Path":"$.xuint16"}},' |
| 139 |
' { "column" : "xuint32", "datatype" : "long", "Properties":{"Path":"$.xuint32"}},' |
| 140 |
' { "column" : "xuint64", "datatype" : "long", "Properties":{"Path":"$.xuint64"}},' |
| 141 |
' { "column" : "xdate", "datatype" : "datetime", "Properties":{"Path":"$.xdate"}},' |
| 142 |
' { "column" : "xsmalltext", "datatype" : "string", "Properties":{"Path":"$.xsmalltext"}},' |
| 143 |
' { "column" : "xtext", "datatype" : "string", "Properties":{"Path":"$.xtext"}},' |
| 144 |
' { "column" : "xnumberAsText", "datatype" : "string", "Properties":{"Path":"$.rowguid"}},' |
| 145 |
' { "column" : "xtime", "datatype" : "timespan", "Properties":{"Path":"$.xtime"}},' |
| 146 |
' { "column" : "xtextWithNulls", "datatype" : "string", "Properties":{"Path":"$.xtextWithNulls"}},' |
| 147 |
' { "column" : "xdynamicWithNulls", "datatype" : "dynamic", "Properties":{"Path":"$.xdynamicWithNulls"}},' |
| 148 |
']'""" |
| 149 |
|
| 150 |
@staticmethod |
| 151 |
def get_file_path() -> str: |
| 152 |
current_dir = os.getcwd() |
| 153 |
path_parts = ["azure-kusto-ingest", "tests", "input"] |
| 154 |
missing_path_parts = [] |
| 155 |
for path_part in path_parts: |
| 156 |
if path_part not in current_dir: |
| 157 |
missing_path_parts.append(path_part) |
| 158 |
return os.path.join(current_dir, *missing_path_parts) |
| 159 |
|
| 160 |
@classmethod |
| 161 |
def engine_kcsb_from_env(cls, is_async=False) -> KustoConnectionStringBuilder: |
| 162 |
return KustoConnectionStringBuilder.with_azure_token_credential( |
| 163 |
cls.engine_cs, |
| 164 |
credential=cls.cred() if not is_async else cls.async_cred(), |
| 165 |
) |
| 166 |
|
| 167 |
@classmethod |
| 168 |
def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder: |
| 169 |
return KustoConnectionStringBuilder.with_azure_token_credential(cls.dm_cs, credential=cls.cred()) |
| 170 |
|
| 171 |
@classmethod |
| 172 |
def setup_class(cls): |
| 173 |
# DM CS can be composed from engine CS |
| 174 |
cls.engine_cs = get_env("ENGINE_CONNECTION_STRING") |
| 175 |
cls.dm_cs = get_env("DM_CONNECTION_STRING", default=cls.engine_cs.replace("//", "//ingest-")) |
| 176 |
|
| 177 |
# Called to set the env variables for the default azure credentials |
| 178 |
prepare_app_key_auth(optional=True) |
| 179 |
|
| 180 |
cls.test_db = get_env("TEST_DATABASE") |
| 181 |
cls.test_blob = get_env("TEST_BLOB", optional=True) |
| 182 |
|
| 183 |
cls.cred = lambda: DefaultAzureCredential(exclude_interactive_browser_credential=False) |
| 184 |
# Async credentials don't support interactive browser authentication for now, so until they do, we'll use the sync default credential for async tests |
| 185 |
cls.async_cred = lambda: DefaultAzureCredential(exclude_interactive_browser_credential=False) |
| 186 |
|
| 187 |
# Init clients |
| 188 |
python_version = "_".join([str(v) for v in sys.version_info[:3]]) |
| 189 |
cls.test_table = "python_test_{0}_{1}_{2}".format(python_version, str(int(time.time())), random.randint(1, 100000)) |
| 190 |
|
| 191 |
cls.client = KustoClient(cls.engine_kcsb_from_env()) |
| 192 |
cls.ingest_client = QueuedIngestClient(cls.dm_kcsb_from_env()) |
| 193 |
cls.streaming_ingest_client = KustoStreamingIngestClient(cls.engine_kcsb_from_env()) |
| 194 |
cls.managed_streaming_ingest_client = ManagedStreamingIngestClient(cls.engine_kcsb_from_env(), cls.dm_kcsb_from_env()) |
| 195 |
|
| 196 |
cls.input_folder_path = cls.get_file_path() |
| 197 |
|
| 198 |
cls.csv_file_path = os.path.join(cls.input_folder_path, "dataset.csv") |
| 199 |
cls.tsv_file_path = os.path.join(cls.input_folder_path, "dataset.tsv") |
| 200 |
cls.zipped_csv_file_path = os.path.join(cls.input_folder_path, "dataset.csv.gz") |
| 201 |
cls.json_file_path = os.path.join(cls.input_folder_path, "dataset.json") |
| 202 |
cls.zipped_json_file_path = os.path.join(cls.input_folder_path, "dataset.jsonz.gz") |
| 203 |
|
| 204 |
cls.current_count = 0 |
| 205 |
|
| 206 |
cls.client.execute( |
| 207 |
cls.test_db, |
| 208 |
f".create table {cls.test_table} (rownumber: int, rowguid: string, xdouble: real, xfloat: real, xbool: bool, xint16: int, xint32: int, xint64: long, xuint8: long, xuint16: long, xuint32: long, xuint64: long, xdate: datetime, xsmalltext: string, xtext: string, xnumberAsText: string, xtime: timespan, xtextWithNulls: string, xdynamicWithNulls: dynamic)", |
| 209 |
) |
| 210 |
cls.client.execute(cls.test_db, f".create table {cls.test_table} ingestion json mapping 'JsonMapping' {cls.table_json_mapping_reference()}") |
| 211 |
|
| 212 |
cls.client.execute(cls.test_db, f".alter table {cls.test_table} policy streamingingestion enable ") |
| 213 |
|
| 214 |
# Clear the cache to guarantee that subsequent streaming ingestion requests incorporate database and table schema changes |
| 215 |
# See https://docs.microsoft.com/azure/data-explorer/kusto/management/data-ingestion/clear-schema-cache-command |
| 216 |
cls.client.execute(cls.test_db, ".clear database cache streamingingestion schema") |
| 217 |
|
| 218 |
@classmethod |
| 219 |
def teardown_class(cls): |
| 220 |
cls.client.execute(cls.test_db, ".drop table {} ifexists".format(cls.test_table)) |
| 221 |
cls.client.close() |
| 222 |
cls.ingest_client.close() |
| 223 |
cls.streaming_ingest_client.close() |
| 224 |
cls.managed_streaming_ingest_client.close() |
| 225 |
|
| 226 |
@classmethod |
| 227 |
async def get_async_client(cls) -> AsyncKustoClient: |
| 228 |
return AsyncKustoClient(cls.engine_kcsb_from_env(is_async=True)) |
| 229 |
|
| 230 |
# assertions |
| 231 |
@classmethod |
| 232 |
async def assert_rows_added(cls, expected: int, timeout=60): |
| 233 |
actual = 0 |
| 234 |
while timeout > 0: |
| 235 |
time.sleep(1) |
| 236 |
timeout -= 1 |
| 237 |
|
| 238 |
try: |
| 239 |
command = "{} | count".format(cls.test_table) |
| 240 |
response = cls.client.execute(cls.test_db, command) |
| 241 |
async_client = await cls.get_async_client() |
| 242 |
response_from_async = await async_client.execute(cls.test_db, command) |
| 243 |
except KustoServiceError: |
| 244 |
continue |
| 245 |
|
| 246 |
if response is not None: |
| 247 |
row = response.primary_results[0][0] |
| 248 |
row_async = response_from_async.primary_results[0][0] |
| 249 |
actual = int(row["Count"]) - cls.current_count |
| 250 |
# this is done to allow for data to arrive properly |
| 251 |
if actual >= expected: |
| 252 |
assert row_async == row, "Mismatch answers between async('{0}') and sync('{1}') clients".format(row_async, row) |
| 253 |
break |
| 254 |
|
| 255 |
cls.current_count += actual |
| 256 |
assert actual == expected, "Row count expected = {0}, while actual row count = {1}".format(expected, actual) |
| 257 |
|
| 258 |
@pytest.mark.asyncio |
| 259 |
async def test_csv_ingest_existing_table(self, is_managed_streaming): |
| 260 |
csv_ingest_props = IngestionProperties( |
| 261 |
self.test_db, |
| 262 |
self.test_table, |
| 263 |
data_format=DataFormat.CSV, |
| 264 |
column_mappings=self.get_test_table_csv_mappings(), |
| 265 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 266 |
flush_immediately=True, |
| 267 |
) |
| 268 |
|
| 269 |
client = self.streaming_ingest_client if is_managed_streaming else self.ingest_client |
| 270 |
|
| 271 |
for f in [self.csv_file_path, self.zipped_csv_file_path]: |
| 272 |
client.ingest_from_file(f, csv_ingest_props) |
| 273 |
|
| 274 |
await self.assert_rows_added(20) |
| 275 |
|
| 276 |
@pytest.mark.asyncio |
| 277 |
async def test_csv_ingest_ignore_first_record(self, is_managed_streaming): |
| 278 |
csv_ingest_props = IngestionProperties( |
| 279 |
self.test_db, |
| 280 |
self.test_table, |
| 281 |
data_format=DataFormat.CSV, |
| 282 |
column_mappings=self.get_test_table_csv_mappings(), |
| 283 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 284 |
flush_immediately=True, |
| 285 |
ignore_first_record=True, |
| 286 |
) |
| 287 |
|
| 288 |
for f in [self.csv_file_path, self.zipped_csv_file_path]: |
| 289 |
self.ingest_client.ingest_from_file(f, csv_ingest_props) |
| 290 |
|
| 291 |
await self.assert_rows_added(18) |
| 292 |
|
| 293 |
@pytest.mark.asyncio |
| 294 |
async def test_json_ingest_existing_table(self): |
| 295 |
json_ingestion_props = IngestionProperties( |
| 296 |
self.test_db, |
| 297 |
self.test_table, |
| 298 |
flush_immediately=True, |
| 299 |
data_format=DataFormat.JSON, |
| 300 |
column_mappings=self.table_json_mappings(), |
| 301 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 302 |
) |
| 303 |
|
| 304 |
for f in [self.json_file_path, self.zipped_json_file_path]: |
| 305 |
self.ingest_client.ingest_from_file(f, json_ingestion_props) |
| 306 |
|
| 307 |
await self.assert_rows_added(4) |
| 308 |
|
| 309 |
@pytest.mark.asyncio |
| 310 |
async def test_json_ingest_existing_table_no_mapping(self): |
| 311 |
json_ingestion_props = IngestionProperties( |
| 312 |
self.test_db, |
| 313 |
self.test_table, |
| 314 |
flush_immediately=True, |
| 315 |
data_format=DataFormat.JSON, |
| 316 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 317 |
) |
| 318 |
|
| 319 |
for f in [self.json_file_path, self.zipped_json_file_path]: |
| 320 |
self.ingest_client.ingest_from_file(f, json_ingestion_props) |
| 321 |
|
| 322 |
await self.assert_rows_added(4) |
| 323 |
|
| 324 |
@pytest.mark.asyncio |
| 325 |
async def test_ingest_complicated_props(self): |
| 326 |
validation_policy = ValidationPolicy( |
| 327 |
validation_options=ValidationOptions.ValidateCsvInputConstantColumns, validation_implications=ValidationImplications.Fail |
| 328 |
) |
| 329 |
json_ingestion_props = IngestionProperties( |
| 330 |
self.test_db, |
| 331 |
self.test_table, |
| 332 |
data_format=DataFormat.JSON, |
| 333 |
column_mappings=self.table_json_mappings(), |
| 334 |
additional_tags=["a", "b"], |
| 335 |
ingest_if_not_exists=["aaaa", "bbbb"], |
| 336 |
ingest_by_tags=["ingestByTag"], |
| 337 |
drop_by_tags=["drop", "drop-by"], |
| 338 |
flush_immediately=False, |
| 339 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 340 |
report_method=ReportMethod.Queue, |
| 341 |
validation_policy=validation_policy, |
| 342 |
) |
| 343 |
|
| 344 |
file_paths = [self.json_file_path, self.zipped_json_file_path] |
| 345 |
fds = [FileDescriptor(fp, 0, uuid.uuid4()) for fp in file_paths] |
| 346 |
|
| 347 |
for fd in fds: |
| 348 |
self.ingest_client.ingest_from_file(fd, json_ingestion_props) |
| 349 |
|
| 350 |
await self.assert_rows_added(4) |
| 351 |
|
| 352 |
@pytest.mark.asyncio |
| 353 |
async def test_ingest_from_stream(self, is_managed_streaming): |
| 354 |
validation_policy = ValidationPolicy( |
| 355 |
validation_options=ValidationOptions.ValidateCsvInputConstantColumns, validation_implications=ValidationImplications.Fail |
| 356 |
) |
| 357 |
json_ingestion_props = IngestionProperties( |
| 358 |
self.test_db, |
| 359 |
self.test_table, |
| 360 |
data_format=DataFormat.JSON, |
| 361 |
column_mappings=self.table_json_mappings(), |
| 362 |
additional_tags=["a", "b"], |
| 363 |
ingest_if_not_exists=["aaaa", "bbbb"], |
| 364 |
ingest_by_tags=["ingestByTag"], |
| 365 |
drop_by_tags=["drop", "drop-by"], |
| 366 |
flush_immediately=False, |
| 367 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 368 |
report_method=ReportMethod.Queue, |
| 369 |
validation_policy=validation_policy, |
| 370 |
) |
| 371 |
text = io.StringIO(pathlib.Path(self.json_file_path).read_text()) |
| 372 |
zipped = io.BytesIO(pathlib.Path(self.zipped_json_file_path).read_bytes()) |
| 373 |
|
| 374 |
client = self.managed_streaming_ingest_client if is_managed_streaming else self.ingest_client |
| 375 |
|
| 376 |
client.ingest_from_stream(text, json_ingestion_props) |
| 377 |
client.ingest_from_stream(StreamDescriptor(zipped, is_compressed=True), json_ingestion_props) |
| 378 |
|
| 379 |
await self.assert_rows_added(4) |
| 380 |
|
| 381 |
@pytest.mark.asyncio |
| 382 |
async def test_json_ingestion_ingest_by_tag(self): |
| 383 |
json_ingestion_props = IngestionProperties( |
| 384 |
self.test_db, |
| 385 |
self.test_table, |
| 386 |
data_format=DataFormat.JSON, |
| 387 |
column_mappings=self.table_json_mappings(), |
| 388 |
ingest_if_not_exists=["ingestByTag"], |
| 389 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 390 |
drop_by_tags=["drop", "drop-by"], |
| 391 |
flush_immediately=True, |
| 392 |
) |
| 393 |
|
| 394 |
for f in [self.json_file_path, self.zipped_json_file_path]: |
| 395 |
self.ingest_client.ingest_from_file(f, json_ingestion_props) |
| 396 |
|
| 397 |
await self.assert_rows_added(0) |
| 398 |
|
| 399 |
@pytest.mark.asyncio |
| 400 |
async def test_tsv_ingestion_csv_mapping(self): |
| 401 |
tsv_ingestion_props = IngestionProperties( |
| 402 |
self.test_db, |
| 403 |
self.test_table, |
| 404 |
flush_immediately=True, |
| 405 |
data_format=DataFormat.TSV, |
| 406 |
column_mappings=self.get_test_table_csv_mappings(), |
| 407 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 408 |
) |
| 409 |
|
| 410 |
self.ingest_client.ingest_from_file(self.tsv_file_path, tsv_ingestion_props) |
| 411 |
|
| 412 |
await self.assert_rows_added(10) |
| 413 |
|
| 414 |
@pytest.mark.asyncio |
| 415 |
async def test_ingest_blob(self): |
| 416 |
if not self.test_blob: |
| 417 |
pytest.skip("Provide blob SAS uri with 'dataset.csv'") |
| 418 |
|
| 419 |
csv_ingest_props = IngestionProperties( |
| 420 |
self.test_db, |
| 421 |
self.test_table, |
| 422 |
data_format=DataFormat.CSV, |
| 423 |
column_mappings=self.get_test_table_csv_mappings(), |
| 424 |
report_level=ReportLevel.FailuresAndSuccesses, |
| 425 |
flush_immediately=True, |
| 426 |
) |
| 427 |
|
| 428 |
blob_len = 1578 |
| 429 |
self.ingest_client.ingest_from_blob(BlobDescriptor(self.test_blob, blob_len), csv_ingest_props) |
| 430 |
|
| 431 |
await self.assert_rows_added(10) |
| 432 |
|
| 433 |
# Don't provide size hint |
| 434 |
self.ingest_client.ingest_from_blob(BlobDescriptor(self.test_blob, size=None), csv_ingest_props) |
| 435 |
|
| 436 |
await self.assert_rows_added(10) |
| 437 |
|
| 438 |
@pytest.mark.asyncio |
| 439 |
async def test_streaming_ingest_from_opened_file(self, is_managed_streaming): |
| 440 |
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, data_format=DataFormat.CSV) |
| 441 |
|
| 442 |
client = self.managed_streaming_ingest_client if is_managed_streaming else self.streaming_ingest_client |
| 443 |
with open(self.csv_file_path, "r") as stream: |
| 444 |
client.ingest_from_stream(stream, ingestion_properties=ingestion_properties) |
| 445 |
|
| 446 |
await self.assert_rows_added(10, timeout=120) |
| 447 |
|
| 448 |
@pytest.mark.asyncio |
| 449 |
async def test_streaming_ingest_from_csv_file(self): |
| 450 |
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, flush_immediately=True, data_format=DataFormat.CSV) |
| 451 |
|
| 452 |
for f in [self.csv_file_path, self.zipped_csv_file_path]: |
| 453 |
self.streaming_ingest_client.ingest_from_file(f, ingestion_properties=ingestion_properties) |
| 454 |
|
| 455 |
await self.assert_rows_added(20, timeout=120) |
| 456 |
|
| 457 |
@pytest.mark.asyncio |
| 458 |
async def test_streaming_ingest_from_json_file(self): |
| 459 |
ingestion_properties = IngestionProperties( |
| 460 |
database=self.test_db, |
| 461 |
table=self.test_table, |
| 462 |
flush_immediately=True, |
| 463 |
data_format=DataFormat.JSON, |
| 464 |
ingestion_mapping_reference="JsonMapping", |
| 465 |
ingestion_mapping_kind=IngestionMappingKind.JSON, |
| 466 |
) |
| 467 |
|
| 468 |
for f in [self.json_file_path, self.zipped_json_file_path]: |
| 469 |
self.streaming_ingest_client.ingest_from_file(f, ingestion_properties=ingestion_properties) |
| 470 |
|
| 471 |
await self.assert_rows_added(4, timeout=120) |
| 472 |
|
| 473 |
@pytest.mark.asyncio |
| 474 |
async def test_streaming_ingest_from_csv_io_streams(self): |
| 475 |
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, data_format=DataFormat.CSV) |
| 476 |
byte_sequence = b'0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null' |
| 477 |
bytes_stream = io.BytesIO(byte_sequence) |
| 478 |
self.streaming_ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) |
| 479 |
|
| 480 |
str_sequence = '0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null' |
| 481 |
str_stream = io.StringIO(str_sequence) |
| 482 |
self.streaming_ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) |
| 483 |
|
| 484 |
await self.assert_rows_added(2, timeout=120) |
| 485 |
|
| 486 |
@pytest.mark.asyncio |
| 487 |
async def test_streaming_ingest_from_json_io_streams(self): |
| 488 |
ingestion_properties = IngestionProperties( |
| 489 |
database=self.test_db, |
| 490 |
table=self.test_table, |
| 491 |
data_format=DataFormat.JSON, |
| 492 |
flush_immediately=True, |
| 493 |
ingestion_mapping_reference="JsonMapping", |
| 494 |
ingestion_mapping_kind=IngestionMappingKind.JSON, |
| 495 |
) |
| 496 |
|
| 497 |
byte_sequence = b'{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}' |
| 498 |
bytes_stream = io.BytesIO(byte_sequence) |
| 499 |
self.streaming_ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) |
| 500 |
|
| 501 |
str_sequence = '{"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""}' |
| 502 |
str_stream = io.StringIO(str_sequence) |
| 503 |
self.streaming_ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) |
| 504 |
|
| 505 |
await self.assert_rows_added(2, timeout=120) |
| 506 |
|
| 507 |
@pytest.mark.asyncio |
| 508 |
async def test_streaming_ingest_from_dataframe(self): |
| 509 |
from pandas import DataFrame |
| 510 |
|
| 511 |
fields = [ |
| 512 |
"rownumber", |
| 513 |
"rowguid", |
| 514 |
"xdouble", |
| 515 |
"xfloat", |
| 516 |
"xbool", |
| 517 |
"xint16", |
| 518 |
"xint32", |
| 519 |
"xint64", |
| 520 |
"xunit8", |
| 521 |
"xuint16", |
| 522 |
"xunit32", |
| 523 |
"xunit64", |
| 524 |
"xdate", |
| 525 |
"xsmalltext", |
| 526 |
"xtext", |
| 527 |
"xnumberAsText", |
| 528 |
"xtime", |
| 529 |
"xtextWithNulls", |
| 530 |
"xdynamicWithNulls", |
| 531 |
] |
| 532 |
|
| 533 |
guid = uuid.uuid4() |
| 534 |
|
| 535 |
dynamic_value = ["[email protected]", "[email protected]", "[email protected]"] |
| 536 |
rows = [[0, str(guid), 0.0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, "2014-01-01T01:01:01Z", "Zero", "Zero", "0", "00:00:00", None, dynamic_value]] |
| 537 |
df = DataFrame(data=rows, columns=fields) |
| 538 |
ingestion_properties = IngestionProperties(database=self.test_db, table=self.test_table, flush_immediately=True) |
| 539 |
self.ingest_client.ingest_from_dataframe(df, ingestion_properties) |
| 540 |
|
| 541 |
await self.assert_rows_added(1, timeout=120) |
| 542 |
|
| 543 |
a = self.client.execute(self.test_db, f"{self.test_table} | where rowguid == '{guid}'") |
| 544 |
assert a.primary_results[0].rows[0]["xdynamicWithNulls"] == dynamic_value |
| 545 |
|
| 546 |
@pytest.mark.asyncio |
| 547 |
async def test_streaming_ingest_from_blob(self, is_managed_streaming): |
| 548 |
ingestion_properties = IngestionProperties( |
| 549 |
database=self.test_db, |
| 550 |
table=self.test_table, |
| 551 |
data_format=DataFormat.JSON, |
| 552 |
ingestion_mapping_reference="JsonMapping", |
| 553 |
ingestion_mapping_kind=IngestionMappingKind.JSON, |
| 554 |
) |
| 555 |
export_containers_list = self.ingest_client._resource_manager._kusto_client.execute("NetDefaultDB", ".show export containers") |
| 556 |
containers = [_ResourceUri(s["StorageRoot"]) for s in export_containers_list.primary_results[0]] |
| 557 |
|
| 558 |
for c in containers: |
| 559 |
self.ingest_client._resource_manager._ranked_storage_account_set.add_storage_account(c.storage_account_name) |
| 560 |
|
| 561 |
with FileDescriptor(self.json_file_path).open(False) as stream: |
| 562 |
blob_descriptor = self.ingest_client.upload_blob( |
| 563 |
containers, |
| 564 |
FileDescriptor(self.json_file_path), |
| 565 |
ingestion_properties.database, |
| 566 |
ingestion_properties.table, |
| 567 |
stream, |
| 568 |
None, |
| 569 |
10 * 60, |
| 570 |
3, |
| 571 |
) |
| 572 |
if is_managed_streaming: |
| 573 |
self.managed_streaming_ingest_client.ingest_from_blob(blob_descriptor, ingestion_properties) |
| 574 |
else: |
| 575 |
self.streaming_ingest_client.ingest_from_blob(blob_descriptor, ingestion_properties) |
| 576 |
|
| 577 |
await self.assert_rows_added(2, timeout=120) |