Debian Codegraph @

azure-kusto-ingest/tests/sample.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import io
4
5 from azure.kusto.data import KustoConnectionStringBuilder
6 from azure.kusto.data.data_format import DataFormat
7 from azure.kusto.ingest import (
8 BlobDescriptor,
9 FileDescriptor,
10 IngestionProperties,
11 IngestionStatus,
12 KustoStreamingIngestClient,
13 ManagedStreamingIngestClient,
14 QueuedIngestClient,
15 StreamDescriptor,
16 )
17
18 ##################################################################
19 ## AUTH ##
20 ##################################################################
21
22 cluster = "https://ingest-{cluster_name}.kusto.windows.net/"
23
24 # In case you want to authenticate with AAD application.
25 client_id = "<insert here your AAD application id>"
26 client_secret = "<insert here your AAD application key>"
27
28 # read more at https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id
29 authority_id = "<insert here your tenant id>"
30
31 kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id)
32
33 # In case you want to authenticate with AAD application certificate.
34 filename = "path to a PEM certificate"
35 with open(filename, "r") as pem_file:
36 PEM = pem_file.read()
37
38 thumbprint = "certificate's thumbprint"
39 kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(cluster, client_id, PEM, thumbprint, authority_id)
40
41 # In case you want to authenticate with AAD application certificate Subject Name & Issuer
42 filename = "path to a PEM certificate"
43 with open(filename, "r") as pem_file:
44 PEM = pem_file.read()
45
46 filename = "path to a public certificate"
47 with open(filename, "r") as cert_file:
48 public_certificate = cert_file.read()
49
50 thumbprint = "certificate's thumbprint"
51 kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_sni_authentication(cluster, client_id, PEM, public_certificate, thumbprint, authority_id)
52
53 # In case you want to authenticate with a System Assigned Managed Service Identity (MSI)
54 kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster)
55
56 # In case you want to authenticate with a User Assigned Managed Service Identity (MSI)
57 user_assigned_client_id = "the AAD identity client id"
58 kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster, client_id=user_assigned_client_id)
59
60 # In case you want to authenticate with AAD username and password
61 username = "<username>"
62 password = "<password>"
63 kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, username, password, authority_id)
64
65 # In case you want to authenticate with AAD device code.
66 # Please note that if you choose this option, you'll need to autenticate for every new instance that is initialized.
67 # It is highly recommended to create one instance and use it for all of your queries.
68 kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster)
69
70 # The authentication method will be taken from the chosen KustoConnectionStringBuilder.
71 client = QueuedIngestClient(kcsb)
72
73 # there are more options for authenticating - see azure-kusto-data samples
74
75 ##################################################################
76 ## INGESTION ##
77 ##################################################################
78
79 # there are a lot of useful properties, make sure to go over docs and check them out
80 ingestion_props = IngestionProperties(
81 database="{database_name}",
82 table="{table_name}",
83 data_format=DataFormat.CSV,
84 # in case status update for success are also required (remember to import ReportLevel from azure.kusto.ingest)
85 # report_level=ReportLevel.FailuresAndSuccesses,
86 # in case a mapping is required (remember to import IngestionMappingKind from azure.kusto.data.data_format)
87 # ingestion_mapping_reference="{json_mapping_that_already_exists_on_table}",
88 # ingestion_mapping_kind= IngestionMappingKind.JSON,
89 )
90
91 # ingest from file
92 file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes.
93 client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_props)
94 result = client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_props)
95
96 # Inspect the result for useful information, such as source_id and blob_url
97 print(repr(result))
98
99 # ingest from blob
100 blob_descriptor = BlobDescriptor(
101 "https://{path_to_blob}.csv.gz?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
102 10,
103 ) # 10 is the raw size of the data in bytes.
104 client.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_props)
105
106 # ingest from dataframe
107 import pandas
108
109 fields = ["id", "name", "value"]
110 rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
111
112 df = pandas.DataFrame(data=rows, columns=fields)
113
114 client.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
115
116 # ingest a whole folder.
117 import os
118
119 path = "folder/path"
120 [client.ingest_from_file(f, ingestion_properties=ingestion_props) for f in os.listdir(path)]
121
122 ##################################################################
123 ## INGESTION STATUS ##
124 ##################################################################
125
126 # if status updates are required, something like this can be done
127 import pprint
128 import time
129 from azure.kusto.ingest.status import KustoIngestStatusQueues
130
131 qs = KustoIngestStatusQueues(client)
132
133 MAX_BACKOFF = 180
134
135 backoff = 1
136 while True:
137 ################### NOTICE ####################
138 # in order to get success status updates,
139 # make sure ingestion properties set the
140 # reportLevel=ReportLevel.FailuresAndSuccesses.
141 if qs.success.is_empty() and qs.failure.is_empty():
142 time.sleep(backoff)
143 backoff = min(backoff * 2, MAX_BACKOFF)
144 print("No new messages. backing off for {} seconds".format(backoff))
145 continue
146
147 backoff = 1
148
149 success_messages = qs.success.pop(10)
150 failure_messages = qs.failure.pop(10)
151
152 pprint.pprint("SUCCESS : {}".format(success_messages))
153 pprint.pprint("FAILURE : {}".format(failure_messages))
154
155 # you can of course separate them and dump them into a file for follow up investigations
156 with open("successes.log", "w+") as sf:
157 for sm in success_messages:
158 sf.write(str(sm))
159
160 with open("failures.log", "w+") as ff:
161 for fm in failure_messages:
162 ff.write(str(fm))
163
164 ##################################################################
165 ## STREAMING INGEST ##
166 ##################################################################
167
168 # Authenticate against this cluster endpoint as shows in the Auth section
169 cluster = "https://{cluster_name}.kusto.windows.net"
170
171 client = KustoStreamingIngestClient(kcsb)
172
173 ingestion_properties = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV)
174
175 # ingest from file
176 file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes.
177 client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties)
178 client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_properties)
179
180 # ingest from dataframe
181 import pandas
182
183 fields = ["id", "name", "value"]
184 rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
185
186 df = pandas.DataFrame(data=rows, columns=fields)
187
188 client.ingest_from_dataframe(df, ingestion_properties=ingestion_properties)
189
190 # ingest from stream
191 byte_sequence = b"56,56,56"
192 bytes_stream = io.BytesIO(byte_sequence)
193 client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
194
195 stream_descriptor = StreamDescriptor(bytes_stream)
196 client.ingest_from_stream(stream_descriptor, ingestion_properties=ingestion_properties)
197
198 str_sequence = "57,57,57"
199 str_stream = io.StringIO(str_sequence)
200 client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
201
202 ##################################################################
203 ## NANAGED STREAMING INGEST ##
204 ##################################################################
205
206 # Managed streaming ingest client will try to use streaming ingestion for performance, but will fall back to queued ingestion if unable.
207 dm_cluster = "https://ingest-{cluster_name}.kusto.windows.net"
208
209 kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(dm_cluster, client_id, client_secret, authority_id)
210
211 # Create it from a dm connection string
212 client = ManagedStreamingIngestClient.from_dm_kcsb(kcsb)
213 # or an engine connection string, like a streaming ingestion client with `from_engine_kcsb`
214 # or provide both: `ManagedStreamingIngestClient(engine_kcsb, dm_kcsb)`
215
216 # use client as you would a streaming or queued ingestion client
217
218 byte_sequence = b"56,56,56"
219 bytes_stream = io.BytesIO(byte_sequence)
220 client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
221
222 ingestion_properties = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV)
223
224 # ingest from file
225 file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes.
226 result = client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties)
227
228 # inspect the result to see what type of ingestion was preformed
229 if result.status == IngestionStatus.QUEUED:
230 # fell back to queued ingestion
231 pass
232
233 # Managed streaming ingest client will fall back to queued if:
234 # - Multiple transient errors were encountered when trying to do streaming ingestion
235 # - The ingestion is too large for streaming ingestion (over 4MB)
236 # - The ingestion is directly for a blob