Debian Codegraph @

azure-kusto-ingest/tests/test_ingestion_blob_info.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import json
4 import re
5 import unittest
6 from uuid import UUID
7
8 from azure.kusto.data.data_format import DataFormat
9 from azure.kusto.ingest import (
10 BlobDescriptor,
11 IngestionProperties,
12 ColumnMapping,
13 ReportLevel,
14 ReportMethod,
15 ValidationPolicy,
16 ValidationOptions,
17 ValidationImplications,
18 )
19 from azure.kusto.ingest.ingestion_blob_info import IngestionBlobInfo
20
21 TIMESTAMP_REGEX = "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}"
22
23
24 class IngestionBlobInfoTest(unittest.TestCase):
25 """Tests serialization of ingestion blob info. This serialization will be queued to the DM."""
26
27 def test_blob_info_csv_mapping(self):
28 """Tests serialization of csv ingestion blob info."""
29 validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
30 column_mapping = ColumnMapping("ColumnName", "cslDataType", ordinal=1)
31
32 properties = IngestionProperties(
33 database="database",
34 table="table",
35 data_format=DataFormat.CSV,
36 column_mappings=[column_mapping],
37 additional_tags=["tag"],
38 ingest_if_not_exists=["ingestIfNotExistTags"],
39 ingest_by_tags=["ingestByTags"],
40 drop_by_tags=["dropByTags"],
41 flush_immediately=True,
42 report_level=ReportLevel.DoNotReport,
43 report_method=ReportMethod.Queue,
44 validation_policy=validation_policy,
45 )
46 blob = BlobDescriptor("somepath", 10)
47 blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
48 self._verify_ingestion_blob_info_result(blob_info.to_json())
49
50 def test_blob_csv_mapping_reference(self):
51 """Tests serialization of ingestion blob info with csv mapping reference."""
52 validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
53 properties = IngestionProperties(
54 database="database",
55 table="table",
56 data_format=DataFormat.CSV,
57 ingestion_mapping_reference="csvMappingReference",
58 additional_tags=["tag"],
59 ingest_if_not_exists=["ingestIfNotExistTags"],
60 ingest_by_tags=["ingestByTags"],
61 drop_by_tags=["dropByTags"],
62 flush_immediately=True,
63 report_level=ReportLevel.DoNotReport,
64 report_method=ReportMethod.Queue,
65 validation_policy=validation_policy,
66 )
67 blob = BlobDescriptor("somepath", 10)
68 blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
69 self._verify_ingestion_blob_info_result(blob_info.to_json())
70
71 def test_blob_info_json_mapping(self):
72 """Tests serialization of json ingestion blob info."""
73 validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
74 properties = IngestionProperties(
75 database="database",
76 table="table",
77 data_format=DataFormat.JSON,
78 column_mappings=[ColumnMapping("ColumnName", "datatype", path="jsonpath")],
79 additional_tags=["tag"],
80 ingest_if_not_exists=["ingestIfNotExistTags"],
81 ingest_by_tags=["ingestByTags"],
82 drop_by_tags=["dropByTags"],
83 flush_immediately=True,
84 report_level=ReportLevel.DoNotReport,
85 report_method=ReportMethod.Queue,
86 validation_policy=validation_policy,
87 )
88 blob = BlobDescriptor("somepath", 10)
89 blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
90 self._verify_ingestion_blob_info_result(blob_info.to_json())
91
92 def test_blob_json_mapping_reference(self):
93 """Tests serialization of ingestion blob info with json mapping reference."""
94 validation_policy = ValidationPolicy(ValidationOptions.ValidateCsvInputConstantColumns, ValidationImplications.BestEffort)
95 properties = IngestionProperties(
96 database="database",
97 table="table",
98 data_format=DataFormat.JSON,
99 ingestion_mapping_reference="jsonMappingReference",
100 additional_tags=["tag"],
101 ingest_if_not_exists=["ingestIfNotExistTags"],
102 ingest_by_tags=["ingestByTags"],
103 drop_by_tags=["dropByTags"],
104 flush_immediately=True,
105 report_level=ReportLevel.DoNotReport,
106 report_method=ReportMethod.Queue,
107 validation_policy=validation_policy,
108 )
109 blob = BlobDescriptor("somepath", 10)
110 blob_info = IngestionBlobInfo(blob, properties, auth_context="authorizationContextText")
111 self._verify_ingestion_blob_info_result(blob_info.to_json())
112
113 def _verify_ingestion_blob_info_result(self, ingestion_blob_info):
114 result = json.loads(ingestion_blob_info)
115 assert result is not None
116 assert isinstance(result, dict)
117 assert result["BlobPath"] == "somepath"
118 assert result["DatabaseName"] == "database"
119 assert result["TableName"] == "table"
120 assert isinstance(result["RawDataSize"], int)
121 assert isinstance(result["IgnoreSizeLimit"], bool)
122 assert isinstance(result["FlushImmediately"], bool)
123 assert isinstance(result["RetainBlobOnSuccess"], bool)
124 assert isinstance(result["ReportMethod"], int)
125 assert isinstance(result["ReportLevel"], int)
126 assert isinstance(UUID(result["Id"]), UUID)
127 assert re.match(TIMESTAMP_REGEX, result["SourceMessageCreationTime"])
128 assert result["AdditionalProperties"]["authorizationContext"] == "authorizationContextText"
129 assert result["AdditionalProperties"]["ingestIfNotExists"] == '["ingestIfNotExistTags"]'
130 assert result["AdditionalProperties"]["ValidationPolicy"] in (
131 '{"ValidationImplications":1,"ValidationOptions":1}',
132 '{"ValidationImplications":ValidationImplications.BestEffort,"ValidationOptions":ValidationOptions.ValidateCsvInputConstantColumns}',
133 )
134
135 assert result["AdditionalProperties"]["tags"] == '["tag","drop-by:dropByTags","ingest-by:ingestByTags"]'