| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
from enum import Enum, IntEnum |
| 4 |
from typing import List, Optional, Dict |
| 5 |
|
| 6 |
from azure.kusto.data.data_format import DataFormat, IngestionMappingKind |
| 7 |
from .exceptions import KustoDuplicateMappingError, KustoMissingMappingError, KustoMappingError |
| 8 |
|
| 9 |
|
| 10 |
class ValidationOptions(IntEnum): |
| 11 |
"""Validation options to ingest command.""" |
| 12 |
|
| 13 |
DoNotValidate = 0 |
| 14 |
ValidateCsvInputConstantColumns = 1 |
| 15 |
ValidateCsvInputColumnLevelOnly = 2 |
| 16 |
|
| 17 |
|
| 18 |
class ValidationImplications(IntEnum): |
| 19 |
"""Validation implications to ingest command.""" |
| 20 |
|
| 21 |
Fail = 0 |
| 22 |
BestEffort = 1 |
| 23 |
|
| 24 |
|
| 25 |
class ValidationPolicy: |
| 26 |
"""Validation policy to ingest command.""" |
| 27 |
|
| 28 |
def __init__(self, validation_options=ValidationOptions.DoNotValidate, validation_implications=ValidationImplications.BestEffort): |
| 29 |
self.ValidationOptions = validation_options |
| 30 |
self.ValidationImplications = validation_implications |
| 31 |
|
| 32 |
|
| 33 |
class ReportLevel(IntEnum): |
| 34 |
"""Report level to ingest command.""" |
| 35 |
|
| 36 |
FailuresOnly = 0 |
| 37 |
DoNotReport = 1 |
| 38 |
FailuresAndSuccesses = 2 |
| 39 |
|
| 40 |
|
| 41 |
class ReportMethod(IntEnum): |
| 42 |
"""Report method to ingest command.""" |
| 43 |
|
| 44 |
Queue = 0 |
| 45 |
|
| 46 |
|
| 47 |
class TransformationMethod(Enum): |
| 48 |
"""Transformations to configure over json column mapping |
| 49 |
To read more about mapping transformations look here: https://docs.microsoft.com/en-us/azure/kusto/management/mappings#mapping-transformations""" |
| 50 |
|
| 51 |
NONE = "None" |
| 52 |
PROPERTY_BAG_ARRAY_TO_DICTIONARY = ("PropertyBagArrayToDictionary",) |
| 53 |
SOURCE_LOCATION = "SourceLocation" |
| 54 |
SOURCE_LINE_NUMBER = "SourceLineNumber" |
| 55 |
GET_PATH_ELEMENT = "GetPathElement" |
| 56 |
UNKNOWN_ERROR = "UnknownMethod" |
| 57 |
DATE_TIME_FROM_UNIX_SECONDS = "DateTimeFromUnixSeconds" |
| 58 |
DATE_TIME_FROM_UNIX_MILLISECONDS = "DateTimeFromUnixMilliseconds" |
| 59 |
DATE_TIME_FROM_UNIX_MICROSECONDS = "DateTimeFromUnixMicroseconds" |
| 60 |
DATE_TIME_FROM_UNIX_NANOSECONDS = "DateTimeFromUnixNanoseconds" |
| 61 |
|
| 62 |
|
| 63 |
class ColumnMapping: |
| 64 |
"""Use this class to create mappings for IngestionProperties.ingestionMappings and utilize mappings that were not |
| 65 |
pre-created (it is recommended to create the mappings in advance and use ingestionMappingReference). |
| 66 |
To read more about mappings look here: https://docs.microsoft.com/en-us/azure/kusto/management/mappings""" |
| 67 |
|
| 68 |
PATH = "Path" |
| 69 |
TRANSFORMATION_METHOD = "Transform" |
| 70 |
ORDINAL = "Ordinal" |
| 71 |
CONST_VALUE = "ConstValue" |
| 72 |
FIELD_NAME = "Field" |
| 73 |
|
| 74 |
NEEDED_PROPERTIES: Dict[IngestionMappingKind, List[str]] = { |
| 75 |
IngestionMappingKind.CSV: [ORDINAL, CONST_VALUE], |
| 76 |
IngestionMappingKind.JSON: [PATH, CONST_VALUE, TRANSFORMATION_METHOD], |
| 77 |
IngestionMappingKind.AVRO: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], |
| 78 |
IngestionMappingKind.APACHEAVRO: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], |
| 79 |
IngestionMappingKind.SSTREAM: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], |
| 80 |
IngestionMappingKind.PARQUET: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], |
| 81 |
IngestionMappingKind.ORC: [PATH, CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], |
| 82 |
IngestionMappingKind.W3CLOGFILE: [CONST_VALUE, FIELD_NAME, TRANSFORMATION_METHOD], |
| 83 |
} |
| 84 |
|
| 85 |
CONSTANT_TRANSFORMATION_METHODS = [TransformationMethod.SOURCE_LOCATION.value, TransformationMethod.SOURCE_LINE_NUMBER.value] |
| 86 |
|
| 87 |
# TODO - add safe and convenient ctors, like in node |
| 88 |
def __init__( |
| 89 |
self, |
| 90 |
column_name: str, |
| 91 |
column_type: str, |
| 92 |
path: str = None, |
| 93 |
transform: TransformationMethod = TransformationMethod.NONE, |
| 94 |
ordinal: int = None, |
| 95 |
const_value: str = None, |
| 96 |
field=None, |
| 97 |
columns=None, |
| 98 |
storage_data_type=None, |
| 99 |
): |
| 100 |
""" |
| 101 |
:param columns: Deprecated. Columns is not used anymore. |
| 102 |
:param storage_data_type: Deprecated. StorageDataType is not used anymore. |
| 103 |
""" |
| 104 |
self.column = column_name |
| 105 |
self.datatype = column_type |
| 106 |
self.properties = {} |
| 107 |
if path: |
| 108 |
self.properties[self.PATH] = path |
| 109 |
if transform != TransformationMethod.NONE: |
| 110 |
self.properties[self.TRANSFORMATION_METHOD] = transform.value |
| 111 |
if ordinal is not None: |
| 112 |
self.properties[self.ORDINAL] = str(ordinal) |
| 113 |
if const_value: |
| 114 |
self.properties[self.CONST_VALUE] = const_value |
| 115 |
if field: |
| 116 |
self.properties[self.FIELD_NAME] = field |
| 117 |
|
| 118 |
def is_valid(self, kind: IngestionMappingKind) -> (bool, List[str]): |
| 119 |
if not self.column: |
| 120 |
return False, ["Column name is required"] |
| 121 |
|
| 122 |
results = [] |
| 123 |
|
| 124 |
needed_props = self.NEEDED_PROPERTIES[kind] |
| 125 |
|
| 126 |
if all(prop not in self.properties for prop in needed_props): |
| 127 |
results.append(f"{kind} needs at least one of the required properties: {needed_props}") |
| 128 |
|
| 129 |
if self.properties.get(self.TRANSFORMATION_METHOD): |
| 130 |
if (self.properties.get(self.PATH) or self.properties.get(self.FIELD_NAME)) and self.properties.get( |
| 131 |
self.TRANSFORMATION_METHOD |
| 132 |
) in self.CONSTANT_TRANSFORMATION_METHODS: |
| 133 |
results.append( |
| 134 |
f"When specifying {self.PATH} or {self.FIELD_NAME}, {self.TRANSFORMATION_METHOD} must not be one of " |
| 135 |
f"{','.join(str(x) for x in self.CONSTANT_TRANSFORMATION_METHODS)}, not {self.properties.get(self.TRANSFORMATION_METHOD)}." |
| 136 |
) |
| 137 |
|
| 138 |
if (not self.properties.get(self.PATH) and not self.properties.get(self.FIELD_NAME)) and self.properties.get( |
| 139 |
self.TRANSFORMATION_METHOD |
| 140 |
) not in self.CONSTANT_TRANSFORMATION_METHODS: |
| 141 |
results.append( |
| 142 |
f"When not specifying {self.PATH} or {self.FIELD_NAME}, {self.TRANSFORMATION_METHOD} must be one of" |
| 143 |
f" {','.join(str(x) for x in self.CONSTANT_TRANSFORMATION_METHODS)}, not {self.properties.get(self.TRANSFORMATION_METHOD)}." |
| 144 |
) |
| 145 |
|
| 146 |
return not bool(results), results |
| 147 |
|
| 148 |
|
| 149 |
class IngestionProperties: |
| 150 |
""" |
| 151 |
Class to represent ingestion properties. |
| 152 |
For more information check out https://docs.microsoft.com/en-us/azure/data-explorer/ingestion-properties |
| 153 |
""" |
| 154 |
|
| 155 |
_DATABASE = "database" |
| 156 |
_TABLE = "table" |
| 157 |
|
| 158 |
def __init__( |
| 159 |
self, |
| 160 |
database: str, |
| 161 |
table: str, |
| 162 |
data_format: DataFormat = DataFormat.CSV, |
| 163 |
column_mappings: Optional[List[ColumnMapping]] = None, |
| 164 |
ingestion_mapping_kind: Optional[IngestionMappingKind] = None, |
| 165 |
ingestion_mapping_reference: Optional[str] = None, |
| 166 |
ingest_if_not_exists: Optional[List[str]] = None, |
| 167 |
ingest_by_tags: Optional[List[str]] = None, |
| 168 |
drop_by_tags: Optional[List[str]] = None, |
| 169 |
additional_tags: Optional[List[str]] = None, |
| 170 |
flush_immediately: bool = False, |
| 171 |
ignore_first_record: bool = False, |
| 172 |
report_level: ReportLevel = ReportLevel.DoNotReport, |
| 173 |
report_method: ReportMethod = ReportMethod.Queue, |
| 174 |
validation_policy: Optional[ValidationPolicy] = None, |
| 175 |
additional_properties: Optional[dict] = None, |
| 176 |
): |
| 177 |
if ingestion_mapping_reference is None and column_mappings is None: |
| 178 |
if ingestion_mapping_kind is not None: |
| 179 |
raise KustoMissingMappingError(f"When ingestion mapping kind is set ('{ingestion_mapping_kind.value}'), a mapping must be provided.") |
| 180 |
else: # A mapping is provided |
| 181 |
if ingestion_mapping_kind is not None: |
| 182 |
if data_format.ingestion_mapping_kind != ingestion_mapping_kind: |
| 183 |
raise KustoMappingError( |
| 184 |
f"Wrong ingestion mapping for format '{data_format.kusto_value}'; mapping kind should be '{data_format.ingestion_mapping_kind.value}', " |
| 185 |
f"but was '{ingestion_mapping_kind.value}'. " |
| 186 |
) |
| 187 |
else: |
| 188 |
ingestion_mapping_kind = data_format.ingestion_mapping_kind |
| 189 |
|
| 190 |
if column_mappings is not None: |
| 191 |
if ingestion_mapping_reference is not None: |
| 192 |
raise KustoDuplicateMappingError() |
| 193 |
|
| 194 |
validation_errors = [] |
| 195 |
|
| 196 |
for mapping in column_mappings: |
| 197 |
(valid, mapping_errors) = mapping.is_valid(ingestion_mapping_kind) |
| 198 |
if not valid: |
| 199 |
validation_errors.extend(f"Column mapping '{mapping.column}' is invalid - '{e}'" for e in mapping_errors) |
| 200 |
|
| 201 |
if validation_errors: |
| 202 |
errors = "\n".join(validation_errors) |
| 203 |
raise KustoMappingError(f"Failed with validation errors:\n{errors}") |
| 204 |
|
| 205 |
self.database = database |
| 206 |
self.table = table |
| 207 |
self.format = data_format |
| 208 |
self.ingestion_mapping = column_mappings |
| 209 |
self.ingestion_mapping_type = ingestion_mapping_kind |
| 210 |
self.ingestion_mapping_reference = ingestion_mapping_reference |
| 211 |
self.additional_tags = additional_tags |
| 212 |
self.ingest_if_not_exists = ingest_if_not_exists |
| 213 |
self.ingest_by_tags = ingest_by_tags |
| 214 |
self.drop_by_tags = drop_by_tags |
| 215 |
self.flush_immediately = flush_immediately |
| 216 |
self.ignore_first_record = ignore_first_record |
| 217 |
self.report_level = report_level |
| 218 |
self.report_method = report_method |
| 219 |
self.validation_policy = validation_policy |
| 220 |
self.additional_properties = additional_properties |
| 221 |
|
| 222 |
def get_tracing_attributes(self) -> dict: |
| 223 |
"""Gets dictionary of attributes to be documented during tracing""" |
| 224 |
return {self._DATABASE: self.database, self._TABLE: self.table} |