| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
import json |
| 4 |
import re |
| 5 |
import unittest |
| 6 |
from uuid import UUID |
| 7 |
|
| 8 |
from azure.kusto.data.data_format import DataFormat |
| 9 |
from azure.kusto.ingest import ( |
| 10 |
BlobDescriptor, |
| 11 |
IngestionProperties, |
| 12 |
ColumnMapping, |
| 13 |
ReportLevel, |
| 14 |
ReportMethod, |
| 15 |
ValidationPolicy, |
| 16 |
ValidationOptions, |
| 17 |
ValidationImplications, |
| 18 |
) |
| 19 |
from azure.kusto.ingest.ingestion_blob_info import IngestionBlobInfo |
| 20 |
|
| 21 |
TIMESTAMP_REGEX = "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}" |
| 22 |
|
| 23 |
|
| 24 |
class IngestionBlobInfoTest(unittest.TestCase): |
| 25 |
"""Tests serialization of ingestion blob info. This serialization will be queued to the DM.""" |
| 26 |
|
| 27 |
def test_blob_info_csv_mapping(self): |
| 28 |
"""Tests serialization of csv ingestion blob info.""" |
| 29 |
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort) |
| 30 |
column_mapping = ColumnMapping("ColumnName", "cslDataType", ordinal=1) |
| 31 |
|
| 32 |
properties = IngestionProperties( |
| 33 |
database="database", |
| 34 |
table="table", |
| 35 |
data_format=DataFormat.CSV, |
| 36 |
column_mappings=[column_mapping], |
| 37 |
additional_tags=["tag"], |
| 38 |
ingest_if_not_exists=["ingestIfNotExistTags"], |
| 39 |
ingest_by_tags=["ingestByTags"], |
| 40 |
drop_by_tags=["dropByTags"], |
| 41 |
flush_immediately=True, |
| 42 |
report_level=ReportLevel.DoNotReport, |
| 43 |
report_method=ReportMethod.Queue, |
| 44 |
validation_policy=validation_policy, |
| 45 |
) |
| 46 |
blob = BlobDescriptor("somepath", 10) |
| 47 |
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText") |
| 48 |
self._verify_ingestion_blob_info_result(blob_info.to_json()) |
| 49 |
|
| 50 |
def test_blob_csv_mapping_reference(self): |
| 51 |
"""Tests serialization of ingestion blob info with csv mapping reference.""" |
| 52 |
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort) |
| 53 |
properties = IngestionProperties( |
| 54 |
database="database", |
| 55 |
table="table", |
| 56 |
data_format=DataFormat.CSV, |
| 57 |
ingestion_mapping_reference="csvMappingReference", |
| 58 |
additional_tags=["tag"], |
| 59 |
ingest_if_not_exists=["ingestIfNotExistTags"], |
| 60 |
ingest_by_tags=["ingestByTags"], |
| 61 |
drop_by_tags=["dropByTags"], |
| 62 |
flush_immediately=True, |
| 63 |
report_level=ReportLevel.DoNotReport, |
| 64 |
report_method=ReportMethod.Queue, |
| 65 |
validation_policy=validation_policy, |
| 66 |
) |
| 67 |
blob = BlobDescriptor("somepath", 10) |
| 68 |
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText") |
| 69 |
self._verify_ingestion_blob_info_result(blob_info.to_json()) |
| 70 |
|
| 71 |
def test_blob_info_json_mapping(self): |
| 72 |
"""Tests serialization of json ingestion blob info.""" |
| 73 |
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort) |
| 74 |
properties = IngestionProperties( |
| 75 |
database="database", |
| 76 |
table="table", |
| 77 |
data_format=DataFormat.JSON, |
| 78 |
column_mappings=[ColumnMapping("ColumnName", "datatype", path="jsonpath")], |
| 79 |
additional_tags=["tag"], |
| 80 |
ingest_if_not_exists=["ingestIfNotExistTags"], |
| 81 |
ingest_by_tags=["ingestByTags"], |
| 82 |
drop_by_tags=["dropByTags"], |
| 83 |
flush_immediately=True, |
| 84 |
report_level=ReportLevel.DoNotReport, |
| 85 |
report_method=ReportMethod.Queue, |
| 86 |
validation_policy=validation_policy, |
| 87 |
) |
| 88 |
blob = BlobDescriptor("somepath", 10) |
| 89 |
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText") |
| 90 |
self._verify_ingestion_blob_info_result(blob_info.to_json()) |
| 91 |
|
| 92 |
def test_blob_json_mapping_reference(self): |
| 93 |
"""Tests serialization of ingestion blob info with json mapping reference.""" |
| 94 |
validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort) |
| 95 |
properties = IngestionProperties( |
| 96 |
database="database", |
| 97 |
table="table", |
| 98 |
data_format=DataFormat.JSON, |
| 99 |
ingestion_mapping_reference="jsonMappingReference", |
| 100 |
additional_tags=["tag"], |
| 101 |
ingest_if_not_exists=["ingestIfNotExistTags"], |
| 102 |
ingest_by_tags=["ingestByTags"], |
| 103 |
drop_by_tags=["dropByTags"], |
| 104 |
flush_immediately=True, |
| 105 |
report_level=ReportLevel.DoNotReport, |
| 106 |
report_method=ReportMethod.Queue, |
| 107 |
validation_policy=validation_policy, |
| 108 |
) |
| 109 |
blob = BlobDescriptor("somepath", 10) |
| 110 |
blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText") |
| 111 |
self._verify_ingestion_blob_info_result(blob_info.to_json()) |
| 112 |
|
| 113 |
def _verify_ingestion_blob_info_result(self, ingestion_blob_info): |
| 114 |
result = json.loads(ingestion_blob_info) |
| 115 |
assert result is not None |
| 116 |
assert isinstance(result, dict) |
| 117 |
assert result["BlobPath"] == "somepath" |
| 118 |
assert result["DatabaseName"] == "database" |
| 119 |
assert result["TableName"] == "table" |
| 120 |
assert isinstance(result["RawDataSize"], int) |
| 121 |
assert isinstance(result["IgnoreSizeLimit"], bool) |
| 122 |
assert isinstance(result["FlushImmediately"], bool) |
| 123 |
assert isinstance(result["RetainBlobOnSuccess"], bool) |
| 124 |
assert isinstance(result["ReportMethod"], int) |
| 125 |
assert isinstance(result["ReportLevel"], int) |
| 126 |
assert isinstance(UUID(result["Id"]), UUID) |
| 127 |
assert re.match(TIMESTAMP_REGEX, result["SourceMessageCreationTime"]) |
| 128 |
assert result["AdditionalProperties"]["authorizationContext"] == "authorizationContextText" |
| 129 |
assert result["AdditionalProperties"]["ingestIfNotExists"] == '["ingestIfNotExistTags"]' |
| 130 |
assert result["AdditionalProperties"]["ValidationPolicy"] in ( |
| 131 |
'{"ValidationImplications":1,"ValidationOptions":1}', |
| 132 |
'{"ValidationImplications":ValidationImplications.BestEffort,"ValidationOptions":ValidationOptions.ValidateCsvInputConstantColumns}', |
| 133 |
) |
| 134 |
|
| 135 |
assert result["AdditionalProperties"]["tags"] == '["tag","drop-by:dropByTags","ingest-by:ingestByTags"]' |