Debian Codegraph @

quick_start/sample_app.py

raw
1 import dataclasses
2 import enum
3 import json
4 import uuid
5 import os
6 from dataclasses import dataclass
7 from typing import List
8
9 import inflection as inflection
10
11 from azure.kusto.data import DataFormat, KustoClient
12 from azure.kusto.ingest import QueuedIngestClient
13 from utils import AuthenticationModeOptions, Utils
14
15 # Declare OpenTelemetry as enabled tracing plugin for Azure SDKs
16 from azure.core.settings import settings
17 from azure.core.tracing import SpanKind
18 from azure.core.tracing.decorator import distributed_trace
19 from azure.core.tracing.ext.opentelemetry_span import OpenTelemetrySpan
20
21 # See https://github.com/open-telemetry/opentelemetry-python for details on regular open telemetry usage
22 from opentelemetry import trace
23 from opentelemetry.trace import Tracer
24 from opentelemetry.sdk.trace import TracerProvider
25 from opentelemetry.sdk.trace.export import SimpleSpanProcessor
26
27 from opentelemetry.sdk.trace.export import ConsoleSpanExporter
28
29 # alternative azure monitor exporter for distributed tracing
30 from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
31
32
33 def enable_distributed_tracing() -> "Tracer":
34 """
35 Sets up distributed tracing environment for sample app
36
37 Returns a 'Tracer'
38 """
39
40 settings.tracing_implementation = OpenTelemetrySpan
41
42 # In the below example, we use a simple console exporter, but you can use anything OpenTelemetry supports. To use
43 # Azure Monitor Exporter uncomment the lines below and add the 'uuid of the instrumentation key (see your Azure
44 # Monitor account)'.
45
46 exporter = ConsoleSpanExporter()
47 # exporter = AzureMonitorTraceExporter.from_connection_string(conn_str=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"])
48
49 trace.set_tracer_provider(TracerProvider())
50 tracer = trace.get_tracer(__name__)
51
52 span_processor = SimpleSpanProcessor(exporter)
53 tr: TracerProvider = trace.get_tracer_provider()
54 tr.add_span_processor(span_processor)
55 return tracer
56
57
58 class SourceType(enum.Enum):
59 """
60 SourceType - represents the type of files used for ingestion
61 """
62
63 local_file_source = "localFileSource"
64 blob_source = "blobSource"
65 nosource = "nosource"
66
67
68 @dataclass
69 class ConfigData:
70 """
71 ConfigData object - represents a file from which to ingest
72 """
73
74 source_type: SourceType
75 data_source_uri: str
76 data_format: DataFormat
77 use_existing_mapping: bool
78 mapping_name: str
79 mapping_value: str
80
81 @staticmethod
82 def from_json(json_dict: dict) -> "ConfigData":
83 json_dict["sourceType"] = SourceType[inflection.underscore(json_dict.pop("sourceType"))]
84 json_dict["dataFormat"] = DataFormat[json_dict.pop("format").upper()]
85 return data_class_from_json(json_dict, ConfigData)
86
87
88 @dataclass
89 class ConfigJson:
90 """
91 ConfigJson object - represents a cluster and DataBase connection configuration file.
92 """
93
94 use_existing_table: bool
95 database_name: str
96 table_name: str
97 table_schema: str
98 kusto_uri: str
99 ingest_uri: str
100 data_to_ingest: List[ConfigData]
101 alter_table: bool
102 query_data: bool
103 ingest_data: bool
104 authentication_mode: AuthenticationModeOptions
105 wait_for_user: bool
106 ignore_first_record: bool
107 wait_for_ingest_seconds: bool
108 batching_policy: str
109
110 @staticmethod
111 def from_json(json_dict: dict) -> "ConfigJson":
112 json_dict["dataToIngest"] = [ConfigData.from_json(j) for j in json_dict.pop("data")]
113 return data_class_from_json(json_dict, ConfigJson)
114
115
116 def remove_extra_keys(json_dict: dict, data_type: type) -> dict:
117 assert dataclasses.is_dataclass(data_type)
118 field_names = [field.name for field in dataclasses.fields(data_type)]
119 return {key: value for key, value in json_dict.items() if key in field_names}
120
121
122 def keys_to_snake_case(json_dict: dict) -> dict:
123 return {inflection.underscore(key): val for (key, val) in json_dict.items()}
124
125
126 def data_class_from_json(json_dict: dict, data_type: type) -> dataclasses.dataclass:
127 assert dataclasses.is_dataclass(data_type)
128 all_keys = keys_to_snake_case(json_dict)
129 config_json_keys = remove_extra_keys(all_keys, data_type)
130 return data_type(**config_json_keys)
131
132
133 class KustoSampleApp:
134 # TODO (config):
135 # If this quickstart app was downloaded from OneClick, kusto_sample_config.json should be pre-populated with your cluster's details
136 # If this quickstart app was downloaded from GitHub, edit kusto_sample_config.json and modify the cluster URL and database fields appropriately
137 CONFIG_FILE_NAME = "kusto_sample_config.json"
138
139 __step = 1
140 config = None
141
142 @classmethod
143 def load_configs(cls, config_file_name: str) -> None:
144 """
145 Loads JSON configuration file, and sets the metadata in place.
146 :param config_file_name: Configuration file path.
147 """
148 try:
149 with open(config_file_name, "r") as config_file:
150 json_dict = json.load(config_file)
151 cls.config = ConfigJson.from_json(json_dict)
152
153 except Exception as ex:
154 Utils.error_handler(f"Couldn't read load config file from file '{config_file_name}'", ex)
155
156 @classmethod
157 @distributed_trace(kind=SpanKind.CLIENT)
158 def pre_ingestion_querying(cls, config: ConfigJson, kusto_client: KustoClient) -> None:
159 """
160 First phase, pre ingestion - will reach the provided DB with several control commands and a query based on the configuration File.
161 :param config: ConfigJson object containing the SampleApp configuration
162 :param kusto_client: Client to run commands
163 """
164 if config.use_existing_table:
165 if config.alter_table:
166 # Tip: Usually table was originally created with a schema appropriate for the data being ingested, so this wouldn't be needed.
167 # Learn More: For more information about altering table schemas,
168 # see: https://docs.microsoft.com/azure/data-explorer/kusto/management/alter-table-command
169 cls.wait_for_user_to_proceed(f"Alter-merge existing table '{config.database_name}.{config.table_name}' to align with the provided schema")
170 cls.alter_merge_existing_table_to_provided_schema(kusto_client, config.database_name, config.table_name, config.table_schema)
171 if config.query_data:
172 # Learn More: For more information about Kusto Query Language (KQL), see: https://docs.microsoft.com/azure/data-explorer/write-queries
173 cls.wait_for_user_to_proceed(f"Get existing row count in '{config.database_name}.{config.table_name}'")
174 cls.query_existing_number_of_rows(kusto_client, config.database_name, config.table_name)
175 else:
176 # Tip: This is generally a one-time configuration
177 # Learn More: For more information about creating tables, see: https://docs.microsoft.com/azure/data-explorer/one-click-table
178 cls.wait_for_user_to_proceed(f"Create table '{config.database_name}.{config.table_name}'")
179 cls.create_new_table(kusto_client, config.database_name, config.table_name, config.table_schema)
180
181 # Learn More:
182 # Kusto batches data for ingestion efficiency. The default batching policy ingests data when one of the following conditions are met:
183 # 1) More than 1,000 files were queued for ingestion for the same table by the same user
184 # 2) More than 1GB of data was queued for ingestion for the same table by the same user
185 # 3) More than 5 minutes have passed since the first file was queued for ingestion for the same table by the same user
186 # For more information about customizing ingestion batching policy, see: https://docs.microsoft.com/azure/data-explorer/kusto/management/batchingpolicy
187
188 # TODO: Change if needed. Disabled to prevent an existing batching policy from being unintentionally changed
189 if False and config.batching_policy:
190 cls.wait_for_user_to_proceed(f"Alter the batching policy for table '{config.database_name}.{config.table_name}'")
191 cls.alter_batching_policy(kusto_client, config.database_name, config.table_name, config.batching_policy)
192
193 @classmethod
194 def alter_merge_existing_table_to_provided_schema(cls, kusto_client: KustoClient, database_name: str, table_name: str, table_schema: str) -> None:
195 """
196 Alter-merges the given existing table to provided schema.
197 :param kusto_client: Client to run commands
198 :param database_name: DB name
199 :param table_name: Table name
200 :param table_schema: Table Schema
201 """
202 command = f".alter-merge table {table_name} {table_schema}"
203 Utils.Queries.execute_command(kusto_client, database_name, command)
204
205 @classmethod
206 @distributed_trace(name_of_span="KustoSampleApp.query", kind=SpanKind.CLIENT) # We can give similar spans the same name
207 def query_existing_number_of_rows(cls, kusto_client: KustoClient, database_name: str, table_name: str) -> None:
208 """
209 Queries the data on the existing number of rows.
210 :param kusto_client: Client to run commands
211 :param database_name: DB name
212 :param table_name: Table name
213 """
214 command = f"{table_name} | count"
215 Utils.Queries.execute_command(kusto_client, database_name, command)
216
217 @classmethod
218 @distributed_trace(name_of_span="KustoSampleApp.query", kind=SpanKind.CLIENT) # We can give similar spans the same name
219 def query_first_two_rows(cls, kusto_client: KustoClient, database_name: str, table_name: str) -> None:
220 """
221 Queries the first two rows of the table.
222 :param kusto_client: Client to run commands
223 :param database_name: DB name
224 :param table_name: Table name
225 """
226 command = f"{table_name} | take 2"
227 Utils.Queries.execute_command(kusto_client, database_name, command)
228
229 @classmethod
230 def create_new_table(cls, kusto_client: KustoClient, database_name: str, table_name: str, table_schema: str) -> None:
231 """
232 Creates a new table.
233 :param kusto_client: Client to run commands
234 :param database_name: DB name
235 :param table_name: Table name
236 :param table_schema: Table Schema
237 """
238 command = f".create table {table_name} {table_schema}"
239 Utils.Queries.execute_command(kusto_client, database_name, command)
240
241 @classmethod
242 def alter_batching_policy(cls, kusto_client: KustoClient, database_name: str, table_name: str, batching_policy: str) -> None:
243 """
244 Alters the batching policy based on BatchingPolicy in configuration.
245 :param kusto_client: Client to run commands
246 :param database_name: DB name
247 :param table_name: Table name
248 :param batching_policy: Ingestion batching policy
249 """
250 # Tip 1: Though most users should be fine with the defaults, to speed up ingestion, such as during development and in this sample app,
251 # we opt to modify the default ingestion policy to ingest data after at most 10 seconds.
252 # Tip 2: This is generally a one-time configuration.
253 # Tip 3: You can skip the batching for some files using the Flush-Immediately property, though this option should be used with care as it is inefficient
254 command = f".alter table {table_name} policy ingestionbatching @'{batching_policy}'"
255 Utils.Queries.execute_command(kusto_client, database_name, command)
256
257 @classmethod
258 def ingestion(cls, config: ConfigJson, kusto_client: KustoClient, ingest_client: QueuedIngestClient) -> None:
259 """
260 Second phase - The ingestion process.
261 :param config: ConfigJson object
262 :param kusto_client: Client to run commands
263 :param ingest_client: Client to ingest data
264 """
265
266 for data_file in config.data_to_ingest:
267 # Tip: This is generally a one-time configuration.
268 # Learn More: For more information about providing inline mappings and mapping references,
269 # see: https://docs.microsoft.com/azure/data-explorer/kusto/management/mappings
270 cls.create_ingestion_mappings(
271 data_file.use_existing_mapping,
272 kusto_client,
273 config.database_name,
274 config.table_name,
275 data_file.mapping_name,
276 data_file.mapping_value,
277 data_file.data_format,
278 )
279
280 # Learn More: For more information about ingesting data to Kusto in Python, see: https://docs.microsoft.com/azure/data-explorer/python-ingest-data
281 cls.ingest_data(
282 data_file, data_file.data_format, ingest_client, config.database_name, config.table_name, data_file.mapping_name, config.ignore_first_record
283 )
284
285 Utils.Ingestion.wait_for_ingestion_to_complete(config.wait_for_ingest_seconds)
286
287 @classmethod
288 def create_ingestion_mappings(
289 cls,
290 use_existing_mapping: bool,
291 kusto_client: KustoClient,
292 database_name: str,
293 table_name: str,
294 mapping_name: str,
295 mapping_value: str,
296 data_format: DataFormat,
297 ) -> None:
298 """
299 Creates Ingestion Mappings (if required) based on given values.
300 :param use_existing_mapping: Flag noting if we should the existing mapping or create a new one
301 :param kusto_client: Client to run commands
302 :param database_name: DB name
303 :param table_name: Table name
304 :param mapping_name: Desired mapping name
305 :param mapping_value: Values of the new mappings to create
306 :param data_format: Given data format
307 """
308 if use_existing_mapping or not mapping_value:
309 return
310
311 ingestion_mapping_kind = data_format.ingestion_mapping_kind.value.lower()
312 cls.wait_for_user_to_proceed(f"Create a '{ingestion_mapping_kind}' mapping reference named '{mapping_name}'")
313
314 mapping_name = mapping_name if mapping_name else "DefaultQuickstartMapping" + str(uuid.UUID())[:5]
315 mapping_command = f".create-or-alter table {table_name} ingestion {ingestion_mapping_kind} mapping '{mapping_name}' '{mapping_value}'"
316 Utils.Queries.execute_command(kusto_client, database_name, mapping_command)
317
318 @classmethod
319 @distributed_trace(kind=SpanKind.CLIENT)
320 def ingest_data(
321 cls,
322 data_file: ConfigData,
323 data_format: DataFormat,
324 ingest_client: QueuedIngestClient,
325 database_name: str,
326 table_name: str,
327 mapping_name: str,
328 ignore_first_record: bool,
329 ) -> None:
330 """
331 Ingest data from given source.
332 :param data_file: Given data source
333 :param data_format: Given data format
334 :param ingest_client: Client to ingest data
335 :param database_name: DB name
336 :param table_name: Table name
337 :param mapping_name: Desired mapping name
338 :param ignore_first_record: Flag noting whether to ignore the first record in the table
339 """
340 source_type = data_file.source_type
341 source_uri = data_file.data_source_uri
342 cls.wait_for_user_to_proceed(f"Ingest '{source_uri}' from '{source_type.name}'")
343
344 # Tip: When ingesting json files, if each line represents a single-line json, use MULTIJSON format even if the file only contains one line.
345 # If the json contains whitespace formatting, use SINGLEJSON. In this case, only one data row json object is allowed per file.
346 data_format = DataFormat.MULTIJSON if data_format == data_format.JSON else data_format
347
348 # Tip: Kusto's Python SDK can ingest data from files, blobs, open streams and pandas dataframes.
349 # See the SDK's samples and the E2E tests in azure.kusto.ingest for additional references.
350 # Note: No need to add "nosource" option as in that case the "ingestData" flag will be set to false, and it will be imppossible to reach this code segemnt anyway.
351 if source_type == SourceType.local_file_source:
352 Utils.Ingestion.ingest_from_file(ingest_client, database_name, table_name, source_uri, data_format, ignore_first_record, mapping_name)
353 elif source_type == SourceType.blob_source:
354 Utils.Ingestion.ingest_from_blob(ingest_client, database_name, table_name, source_uri, data_format, ignore_first_record, mapping_name)
355 else:
356 Utils.error_handler(f"Unknown source '{source_type}' for file '{source_uri}'")
357
358 @classmethod
359 @distributed_trace(kind=SpanKind.CLIENT)
360 def post_ingestion_querying(cls, kusto_client: KustoClient, database_name: str, table_name: str, config_ingest_data: bool) -> None:
361 """
362 Third and final phase - simple queries to validate the script ran successfully.
363 :param kusto_client: Client to run queries
364 :param database_name: DB Name
365 :param table_name: Table Name
366 :param config_ingest_data: Flag noting whether any data was ingested by the script
367 """
368 optional_post_ingestion_message = "post-ingestion " if config_ingest_data else ""
369
370 # Note: We poll here the ingestion's target table because monitoring successful ingestions is expensive and not recommended.
371 # Instead, the recommended ingestion monitoring approach is to monitor failures.
372 # Learn more: https://docs.microsoft.com/azure/data-explorer/kusto/api/netfx/kusto-ingest-client-status#tracking-ingestion-status-kustoqueuedingestclient
373 # and https://docs.microsoft.com/azure/data-explorer/using-diagnostic-logs
374 cls.wait_for_user_to_proceed(f"Get {optional_post_ingestion_message}row count for '{database_name}.{table_name}':")
375 cls.query_existing_number_of_rows(kusto_client, database_name, table_name)
376
377 cls.wait_for_user_to_proceed(f"Get {optional_post_ingestion_message}row count for '{database_name}.{table_name}':")
378 cls.query_first_two_rows(kusto_client, database_name, table_name)
379
380 @classmethod
381 def wait_for_user_to_proceed(cls, prompt_msg: str) -> None:
382 """
383 Handles UX on prompts and flow of program
384 :param prompt_msg: Prompt to display to user
385 """
386 print()
387 print(f"Step {cls.__step}: {prompt_msg}")
388 cls.__step = cls.__step + 1
389 if cls.config.wait_for_user:
390 input("Press ENTER to proceed with this operation...")
391
392
393 @distributed_trace(name_of_span="KustoSampleApp.main", kind=SpanKind.CLIENT)
394 def main():
395 print("Kusto sample app is starting...")
396
397 app = KustoSampleApp()
398 app.load_configs(app.CONFIG_FILE_NAME)
399
400 if app.config.authentication_mode == "UserPrompt":
401 app.wait_for_user_to_proceed("You will be prompted *twice* for credentials during this script. Please return to the console after authenticating.")
402
403 kusto_connection_string = Utils.Authentication.generate_connection_string(app.config.kusto_uri, app.config.authentication_mode)
404 ingest_connection_string = Utils.Authentication.generate_connection_string(app.config.ingest_uri, app.config.authentication_mode)
405
406 # Tip: Avoid creating a new Kusto/ingest client for each use.Instead, create the clients once and reuse them.
407 if not kusto_connection_string or not ingest_connection_string:
408 Utils.error_handler("Connection String error. Please validate your configuration file.")
409 else:
410 # Make sure to use context manager for the Kusto client, as it will close the underlying resources properly.
411 # Alternatively, you can use the client as a regular Python object and call close() on it.
412 with KustoClient(kusto_connection_string) as kusto_client:
413 with QueuedIngestClient(ingest_connection_string) as ingest_client:
414 app.pre_ingestion_querying(app.config, kusto_client)
415
416 if app.config.ingest_data:
417 app.ingestion(app.config, kusto_client, ingest_client)
418
419 if app.config.query_data:
420 app.post_ingestion_querying(kusto_client, app.config.database_name, app.config.table_name, app.config.ingest_data)
421
422 print("\nKusto sample app done")
423
424
425 if __name__ == "__main__":
426 # Uncomment the lines below to enable distributed tracing
427
428 # tracer = enable_distributed_tracing()
429 # with tracer.start_as_current_span(name="KustoSampleApp"):
430 # main()
431
432 main()