Debian Codegraph @

quick_start/utils.py

raw
1 import enum
2 import os
3 import uuid
4 from time import sleep
5 from tqdm import tqdm
6 from azure.kusto.data import KustoConnectionStringBuilder, ClientRequestProperties, KustoClient, DataFormat
7 from azure.kusto.data.exceptions import KustoClientError, KustoServiceError
8 from azure.kusto.ingest import IngestionProperties, BaseIngestClient, QueuedIngestClient, FileDescriptor, BlobDescriptor
9
10
11 class AuthenticationModeOptions(enum.Enum):
12 """
13 AuthenticationModeOptions - represents the different options to autenticate to the system
14 """
15
16 UserPrompt = ("UserPrompt",)
17 ManagedIdentity = ("ManagedIdentity",)
18 AppKey = ("AppKey",)
19 AppCertificate = "AppCertificate"
20
21
22 class Utils:
23 class Authentication:
24 """
25 Authentication module of Utils - in charge of authenticating the user with the system
26 """
27
28 @classmethod
29 def generate_connection_string(cls, cluster_url: str, authentication_mode: AuthenticationModeOptions) -> KustoConnectionStringBuilder:
30 """
31 Generates Kusto Connection String based on given Authentication Mode.
32 :param cluster_url: Cluster to connect to.
33 :param authentication_mode: User Authentication Mode, Options: (UserPrompt|ManagedIdentity|AppKey|AppCertificate)
34 :return: A connection string to be used when creating a Client
35 """
36 # Learn More: For additional information on how to authorize users and apps in Kusto,
37 # see: https://docs.microsoft.com/azure/data-explorer/manage-database-permissions
38
39 if authentication_mode == AuthenticationModeOptions.UserPrompt.name:
40 # Prompt user for credentials
41 return KustoConnectionStringBuilder.with_interactive_login(cluster_url)
42
43 elif authentication_mode == AuthenticationModeOptions.ManagedIdentity.name:
44 # Authenticate using a System-Assigned managed identity provided to an azure service, or using a User-Assigned managed identity.
45 # For more information, see https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview
46 return cls.create_managed_identity_connection_string(cluster_url)
47
48 elif authentication_mode == AuthenticationModeOptions.AppKey.name:
49 # Learn More: For information about how to procure an AAD Application,
50 # see: https://docs.microsoft.com/azure/data-explorer/provision-azure-ad-app
51 # TODO (config - optional): App ID & tenant, and App Key to authenticate with
52 return KustoConnectionStringBuilder.with_aad_application_key_authentication(
53 cluster_url, os.environ.get("APP_ID"), os.environ.get("APP_KEY"), os.environ.get("APP_TENANT")
54 )
55
56 elif authentication_mode == AuthenticationModeOptions.AppCertificate.name:
57 return cls.create_application_certificate_connection_string(cluster_url)
58
59 else:
60 Utils.error_handler(f"Authentication mode '{authentication_mode}' is not supported")
61
62 @classmethod
63 def create_managed_identity_connection_string(cls, cluster_url: str) -> KustoConnectionStringBuilder:
64 """
65 Generates Kusto Connection String based on 'ManagedIdentity' Authentication Mode.
66 :param cluster_url: Url of cluster to connect to
67 :return: ManagedIdentity Kusto Connection String
68 """
69 # Connect using the system- or user-assigned managed identity (Azure service only)
70 # TODO (config - optional): Managed identity client ID if you are using a user-assigned managed identity
71 client_id = os.environ.get("MANAGED_IDENTITY_CLIENT_ID")
72 return (
73 KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster_url, client_id=client_id)
74 if client_id
75 else KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster_url)
76 )
77
78 @classmethod
79 def create_application_certificate_connection_string(cls, cluster_url: str) -> KustoConnectionStringBuilder:
80 """
81 Generates Kusto Connection String based on 'AppCertificate' Authentication Mode.
82 :param cluster_url: Url of cluster to connect to
83 :return: AppCertificate Kusto Connection String
84 """
85
86 # TODO (config - optional): App ID & tenant, path to public certificate and path to private certificate pem file to authenticate with
87 app_id = os.environ.get("APP_ID")
88 app_tenant = os.environ.get("APP_TENANT")
89 private_key_pem_file_path = os.environ.get("PRIVATE_KEY_PEM_FILE_PATH")
90 cert_thumbprint = os.environ.get("CERT_THUMBPRINT")
91 public_cert_file_path = os.environ.get("PUBLIC_CERT_FILE_PATH") # Only used for "Subject Name and Issuer" auth
92 public_certificate = None
93 pem_certificate = None
94
95 try:
96 with open(private_key_pem_file_path, "r") as pem_file:
97 pem_certificate = pem_file.read()
98 except Exception as ex:
99 Utils.error_handler(f"Failed to load PEM file from {private_key_pem_file_path}", ex)
100
101 if public_cert_file_path:
102 try:
103 with open(public_cert_file_path, "r") as cert_file:
104 public_certificate = cert_file.read()
105 except Exception as ex:
106 Utils.error_handler(f"Failed to load public certificate file from {public_cert_file_path}", ex)
107
108 return KustoConnectionStringBuilder.with_aad_application_certificate_sni_authentication(
109 cluster_url, app_id, pem_certificate, public_certificate, cert_thumbprint, app_tenant
110 )
111 else:
112 return KustoConnectionStringBuilder.with_aad_application_certificate_authentication(
113 cluster_url, app_id, pem_certificate, cert_thumbprint, app_tenant
114 )
115
116 class Queries:
117 """
118 Queries module of Utils - in charge of querying the data - either with management queries, or data queries
119 """
120
121 MGMT_PREFIX = "."
122
123 @classmethod
124 def create_client_request_properties(cls, scope: str, timeout: str = None) -> ClientRequestProperties:
125 """
126 Creates a fitting ClientRequestProperties object, to be used when executing control commands or queries.
127 :param scope: Working scope
128 :param timeout: Requests default timeout
129 :return: ClientRequestProperties object
130 """
131 client_request_properties = ClientRequestProperties()
132 client_request_properties.client_request_id = f"{scope};{str(uuid.uuid4())}"
133 client_request_properties.application = "sample_app.py"
134
135 # Tip: Though uncommon, you can alter the request default command timeout using the below command, e.g. to set the timeout to 10 minutes, use "10m"
136 if timeout:
137 client_request_properties.set_option(ClientRequestProperties.request_timeout_option_name, timeout)
138
139 return client_request_properties
140
141 @classmethod
142 def execute_command(cls, kusto_client: KustoClient, database_name: str, command: str) -> bool:
143 """
144 Executes a Command using a premade client
145 :param kusto_client: Premade client to run Commands. can be either an adminClient or queryClient
146 :param database_name: DB name
147 :param command: The Command to execute
148 :return: True on success, false otherwise
149 """
150 try:
151 if command.startswith(cls.MGMT_PREFIX):
152 client_request_properties = cls.create_client_request_properties("Python_SampleApp_ControlCommand")
153 else:
154 client_request_properties = cls.create_client_request_properties("Python_SampleApp_Query")
155
156 result = kusto_client.execute(database_name, command, client_request_properties)
157 print(f"Response from executed command '{command}':")
158 for row in result.primary_results[0]:
159 print(row.to_list())
160
161 return True
162
163 except KustoClientError as ex:
164 Utils.error_handler(f"Client error while trying to execute command '{command}' on database '{database_name}'", ex)
165 except KustoServiceError as ex:
166 Utils.error_handler(f"Server error while trying to execute command '{command}' on database '{database_name}'", ex)
167 except Exception as ex:
168 Utils.error_handler(f"Unknown error while trying to execute command '{command}' on database '{database_name}'", ex)
169
170 return False
171
172 class Ingestion:
173 """
174 Ingestion module of Utils - in charge of ingesting the given data - based on the configuration file.
175 """
176
177 @classmethod
178 def create_ingestion_properties(
179 cls, database_name: str, table_name: str, data_format: DataFormat, mapping_name: str, ignore_first_record: bool
180 ) -> IngestionProperties:
181 """
182 Creates a fitting KustoIngestionProperties object, to be used when executing ingestion commands.
183 :param database_name: DB name
184 :param table_name: Table name
185 :param data_format: Given data format
186 :param mapping_name: Desired mapping name
187 :param ignore_first_record: Flag noting whether to ignore the first record in the table. only relevant to
188 tabular textual formats (CSV and the likes). for more information
189 please read: https://docs.microsoft.com/en-us/azure/data-explorer/ingestion-properties
190 :return: IngestionProperties object
191 """
192 return IngestionProperties(
193 database=f"{database_name}",
194 table=f"{table_name}",
195 ingestion_mapping_reference=mapping_name,
196 # Learn More: For more information about supported data formats, see: https://docs.microsoft.com/azure/data-explorer/ingestion-supported-formats
197 data_format=data_format,
198 # TODO (config - optional): Setting the ingestion batching policy takes up to 5 minutes to take effect.
199 # We therefore set Flush-Immediately for the sake of the sample, but it generally shouldn't be used in practice.
200 # Comment out the line below after running the sample the first few times.
201 flush_immediately=True,
202 ignore_first_record=ignore_first_record,
203 )
204
205 @classmethod
206 def ingest_from_file(
207 cls,
208 ingest_client: BaseIngestClient,
209 database_name: str,
210 table_name: str,
211 file_path: str,
212 data_format: DataFormat,
213 ignore_first_record,
214 mapping_name: str = None,
215 ) -> None:
216 """
217 Ingest Data from a given file path.
218 :param ingest_client: Client to ingest data
219 :param database_name: DB name
220 :param table_name: Table name
221 :param file_path: File path
222 :param data_format: Given data format
223 :param ignore_first_record: Flag noting whether to ignore the first record in the table. only relevant to
224 tabular textual formats (CSV and the likes). for more information
225 please read: https://docs.microsoft.com/en-us/azure/data-explorer/ingestion-properties
226 :param mapping_name: Desired mapping name
227 """
228 ingestion_properties = cls.create_ingestion_properties(database_name, table_name, data_format, mapping_name, ignore_first_record)
229
230 # Tip 1: For optimal ingestion batching and performance,specify the uncompressed data size in the file descriptor instead of the default below of 0.
231 # Otherwise, the service will determine the file size, requiring an additional s2s call, and may not be accurate for compressed files.
232 # Tip 2: To correlate between ingestion operations in your applications and Kusto, set the source ID and log it somewhere
233 file_descriptor = FileDescriptor(file_path, size=0, source_id=uuid.uuid4())
234 ingest_client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties)
235
236 @classmethod
237 def ingest_from_blob(
238 cls,
239 ingest_client: QueuedIngestClient,
240 database_name: str,
241 table_name: str,
242 blob_url: str,
243 data_format: DataFormat,
244 ignore_first_record: bool,
245 mapping_name: str = None,
246 ) -> None:
247 """
248 Ingest Data from a Blob.
249 :param ingest_client: Client to ingest data
250 :param database_name: DB name
251 :param table_name: Table name
252 :param blob_url: Blob Uri
253 :param data_format: Given data format
254 :param ignore_first_record: Flag noting whether to ignore the first record in the table. only relevant to
255 tabular textual formats (CSV and the likes). for more information
256 please read: https://docs.microsoft.com/en-us/azure/data-explorer/ingestion-properties
257 :param mapping_name: Desired mapping name
258 """
259 ingestion_properties = cls.create_ingestion_properties(database_name, table_name, data_format, mapping_name, ignore_first_record)
260
261 # Tip 1: For optimal ingestion batching and performance,specify the uncompressed data size in the file descriptor instead of the default below of 0.
262 # Otherwise, the service will determine the file size, requiring an additional s2s call, and may not be accurate for compressed files.
263 # Tip 2: To correlate between ingestion operations in your applications and Kusto, set the source ID and log it somewhere
264 blob_descriptor = BlobDescriptor(blob_url, size=0, source_id=str(uuid.uuid4()))
265 ingest_client.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_properties)
266
267 @classmethod
268 def wait_for_ingestion_to_complete(cls, wait_for_ingest_seconds: int) -> None:
269 """
270 Halts the program for WaitForIngestSeconds, allowing the queued ingestion process to complete.
271 :param wait_for_ingest_seconds: Sleep time to allow for queued ingestion to complete.
272 """
273 print(
274 f"Sleeping {wait_for_ingest_seconds} seconds for queued ingestion to complete. Note: This may take longer depending on the file size "
275 f"and ingestion batching policy."
276 )
277
278 for x in tqdm(range(wait_for_ingest_seconds, 0, -1)):
279 sleep(1)
280
281 @staticmethod
282 def error_handler(error: str, e: Exception = None) -> None:
283 """
284 Error handling function. Will mention the appropriate error message (and the exception itself if exists), and will quit the program.
285 :param error: Appropriate error message received from calling function
286 :param e: Thrown exception
287 """
288 print(f"Script failed with error: {error}")
289 if e:
290 print(f"Exception: {e}")
291
292 exit(-1)