Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/ingestion_properties.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 from enum import Enum, IntEnum
4 from typing import List, Optional, Dict
5
6 from azure.kusto.data.data_format import DataFormat, IngestionMappingKind
7 from .exceptions import KustoDuplicateMappingError, KustoMissingMappingError, KustoMappingError
8
9
10 class ValidationOptions(IntEnum):
11 """Validation options to ingest command."""
12
13 DoNotValidate = 0
14 ValidateCsvInputConstantColumns = 1
15 ValidateCsvInputColumnLevelOnly = 2
16
17
18 class ValidationImplications(IntEnum):
19 """Validation implications to ingest command."""
20
21 Fail = 0
22 BestEffort = 1
23
24
25 class ValidationPolicy:
26 """Validation policy to ingest command."""
27
28 def __init__(self, validation_options=ValidationOptions.DoNotValidate, validation_implications=ValidationImplications.BestEffort):
29 self.ValidationOptions = validation_options
30 self.ValidationImplications = validation_implications
31
32
33 class ReportLevel(IntEnum):
34 """Report level to ingest command."""
35
36 FailuresOnly = 0
37 DoNotReport = 1
38 FailuresAndSuccesses = 2
39
40
41 class ReportMethod(IntEnum):
42 """Report method to ingest command."""
43
44 Queue = 0
45
46
47 class TransformationMethod(Enum):
48 """Transformations to configure over json column mapping
49 To read more about mapping transformations look here: https://docs.microsoft.com/en-us/azure/kusto/management/mappings#mapping-transformations"""
50
51 NONE = "None"
52 PROPERTY_BAG_ARRAY_TO_DICTIONARY = ("PropertyBagArrayToDictionary",)
53 SOURCE_LOCATION = "SourceLocation"
54 SOURCE_LINE_NUMBER = "SourceLineNumber"
55 GET_PATH_ELEMENT = "GetPathElement"
56 UNKNOWN_ERROR = "UnknownMethod"
57 DATE_TIME_FROM_UNIX_SECONDS = "DateTimeFromUnixSeconds"
58 DATE_TIME_FROM_UNIX_MILLISECONDS = "DateTimeFromUnixMilliseconds"
59 DATE_TIME_FROM_UNIX_MICROSECONDS = "DateTimeFromUnixMicroseconds"
60 DATE_TIME_FROM_UNIX_NANOSECONDS = "DateTimeFromUnixNanoseconds"
61
62
63 class ColumnMapping:
64 """Use this class to create mappings for IngestionProperties.ingestionMappings and utilize mappings that were not
65 pre-created (it is recommended to create the mappings in advance and use ingestionMappingReference).
66 To read more about mappings look here: https://docs.microsoft.com/en-us/azure/kusto/management/mappings"""
67
68 PATH = "Path"
69 TRANSFORMATION_METHOD = "Transform"
70 ORDINAL = "Ordinal"
71 CONST_VALUE = "ConstValue"
72 FIELD_NAME = "Field"
73
74 NEEDED_PROPERTIES: Dict[IngestionMappingKind, List[str]] = {
75 IngestionMappingKind.CSV: [ORDINAL, CONST_VALUE],
76 IngestionMappingKind.JSON: [PATH, CONST_VALUE, TRANSFORMATION_METHOD],
77 IngestionMappingKind.AVRO: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD],
78 IngestionMappingKind.APACHEAVRO: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD],
79 IngestionMappingKind.SSTREAM: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD],
80 IngestionMappingKind.PARQUET: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD],
81 IngestionMappingKind.ORC: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD],
82 IngestionMappingKind.W3CLOGFILE: [CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD],
83 }
84
85 CONSTANT_TRANSFORMATION_METHODS = [TransformationMethod.SOURCE_LOCATION.value, TransformationMethod.SOURCE_LINE_NUMBER.value]
86
87 # TODO - add safe and convenient ctors, like in node
88 def __init__(
89 self,
90 column_name: str,
91 column_type: str,
92 path: str = None,
93 transform: TransformationMethod = TransformationMethod.NONE,
94 ordinal: int = None,
95 const_value: str = None,
96 field=None,
97 columns=None,
98 storage_data_type=None,
99 ):
100 """
101 :param columns: Deprecated. Columns is not used anymore.
102 :param storage_data_type: Deprecated. StorageDataType is not used anymore.
103 """
104 self.column = column_name
105 self.datatype = column_type
106 self.properties = {}
107 if path:
108 self.properties[self.PATH] = path
109 if transform != TransformationMethod.NONE:
110 self.properties[self.TRANSFORMATION_METHOD] = transform.value
111 if ordinal is not None:
112 self.properties[self.ORDINAL] = str(ordinal)
113 if const_value:
114 self.properties[self.CONST_VALUE] = const_value
115 if field:
116 self.properties[self.FIELD_NAME] = field
117
118 def is_valid(self, kind: IngestionMappingKind) -> (bool, List[str]):
119 if not self.column:
120 return False, ["Column name is required"]
121
122 results = []
123
124 needed_props = self.NEEDED_PROPERTIES[kind]
125
126 if all(prop not in self.properties for prop in needed_props):
127 results.append(f"{kind} needs at least one of the required properties: {needed_props}")
128
129 if self.properties.get(self.TRANSFORMATION_METHOD):
130 if (self.properties.get(self.PATH) or self.properties.get(self.FIELD_NAME)) and self.properties.get(
131 self.TRANSFORMATION_METHOD
132 ) in self.CONSTANT_TRANSFORMATION_METHODS:
133 results.append(
134 f"When specifying {self.PATH} or {self.FIELD_NAME}, {self.TRANSFORMATION_METHOD} must not be one of "
135 f"{','.join(str(x) for x in self.CONSTANT_TRANSFORMATION_METHODS)}, not {self.properties.get(self.TRANSFORMATION_METHOD)}."
136 )
137
138 if (not self.properties.get(self.PATH) and not self.properties.get(self.FIELD_NAME)) and self.properties.get(
139 self.TRANSFORMATION_METHOD
140 ) not in self.CONSTANT_TRANSFORMATION_METHODS:
141 results.append(
142 f"When not specifying {self.PATH} or {self.FIELD_NAME}, {self.TRANSFORMATION_METHOD} must be one of"
143 f" {','.join(str(x) for x in self.CONSTANT_TRANSFORMATION_METHODS)}, not {self.properties.get(self.TRANSFORMATION_METHOD)}."
144 )
145
146 return not bool(results), results
147
148
149 class IngestionProperties:
150 """
151 Class to represent ingestion properties.
152 For more information check out https://docs.microsoft.com/en-us/azure/data-explorer/ingestion-properties
153 """
154
155 _DATABASE = "database"
156 _TABLE = "table"
157
158 def __init__(
159 self,
160 database: str,
161 table: str,
162 data_format: DataFormat = DataFormat.CSV,
163 column_mappings: Optional[List[ColumnMapping]] = None,
164 ingestion_mapping_kind: Optional[IngestionMappingKind] = None,
165 ingestion_mapping_reference: Optional[str] = None,
166 ingest_if_not_exists: Optional[List[str]] = None,
167 ingest_by_tags: Optional[List[str]] = None,
168 drop_by_tags: Optional[List[str]] = None,
169 additional_tags: Optional[List[str]] = None,
170 flush_immediately: bool = False,
171 ignore_first_record: bool = False,
172 report_level: ReportLevel = ReportLevel.DoNotReport,
173 report_method: ReportMethod = ReportMethod.Queue,
174 validation_policy: Optional[ValidationPolicy] = None,
175 additional_properties: Optional[dict] = None,
176 ):
177 if ingestion_mapping_reference is None and column_mappings is None:
178 if ingestion_mapping_kind is not None:
179 raise KustoMissingMappingError(f"When ingestion mapping kind is set ('{ingestion_mapping_kind.value}'), a mapping must be provided.")
180 else: # A mapping is provided
181 if ingestion_mapping_kind is not None:
182 if data_format.ingestion_mapping_kind != ingestion_mapping_kind:
183 raise KustoMappingError(
184 f"Wrong ingestion mapping for format '{data_format.kusto_value}'; mapping kind should be '{data_format.ingestion_mapping_kind.value}', "
185 f"but was '{ingestion_mapping_kind.value}'. "
186 )
187 else:
188 ingestion_mapping_kind = data_format.ingestion_mapping_kind
189
190 if column_mappings is not None:
191 if ingestion_mapping_reference is not None:
192 raise KustoDuplicateMappingError()
193
194 validation_errors = []
195
196 for mapping in column_mappings:
197 (valid, mapping_errors) = mapping.is_valid(ingestion_mapping_kind)
198 if not valid:
199 validation_errors.extend(f"Column mapping '{mapping.column}' is invalid - '{e}'" for e in mapping_errors)
200
201 if validation_errors:
202 errors = "\n".join(validation_errors)
203 raise KustoMappingError(f"Failed with validation errors:\n{errors}")
204
205 self.database = database
206 self.table = table
207 self.format = data_format
208 self.ingestion_mapping = column_mappings
209 self.ingestion_mapping_type = ingestion_mapping_kind
210 self.ingestion_mapping_reference = ingestion_mapping_reference
211 self.additional_tags = additional_tags
212 self.ingest_if_not_exists = ingest_if_not_exists
213 self.ingest_by_tags = ingest_by_tags
214 self.drop_by_tags = drop_by_tags
215 self.flush_immediately = flush_immediately
216 self.ignore_first_record = ignore_first_record
217 self.report_level = report_level
218 self.report_method = report_method
219 self.validation_policy = validation_policy
220 self.additional_properties = additional_properties
221
222 def get_tracing_attributes(self) -> dict:
223 """Gets dictionary of attributes to be documented during tracing"""
224 return {self._DATABASE: self.database, self._TABLE: self.table}