| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
import io |
| 4 |
|
| 5 |
from azure.kusto.data import KustoConnectionStringBuilder |
| 6 |
from azure.kusto.data.data_format import DataFormat |
| 7 |
from azure.kusto.ingest import ( |
| 8 |
BlobDescriptor, |
| 9 |
FileDescriptor, |
| 10 |
IngestionProperties, |
| 11 |
IngestionStatus, |
| 12 |
KustoStreamingIngestClient, |
| 13 |
ManagedStreamingIngestClient, |
| 14 |
QueuedIngestClient, |
| 15 |
StreamDescriptor, |
| 16 |
) |
| 17 |
|
| 18 |
################################################################## |
| 19 |
## AUTH ## |
| 20 |
################################################################## |
| 21 |
|
| 22 |
cluster = "https://ingest-{cluster_name}.kusto.windows.net/" |
| 23 |
|
| 24 |
# In case you want to authenticate with AAD application. |
| 25 |
client_id = "<insert here your AAD application id>" |
| 26 |
client_secret = "<insert here your AAD application key>" |
| 27 |
|
| 28 |
# read more at https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id |
| 29 |
authority_id = "<insert here your tenant id>" |
| 30 |
|
| 31 |
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(cluster, client_id, client_secret, authority_id) |
| 32 |
|
| 33 |
# In case you want to authenticate with AAD application certificate. |
| 34 |
filename = "path to a PEM certificate" |
| 35 |
with open(filename, "r") as pem_file: |
| 36 |
PEM = pem_file.read() |
| 37 |
|
| 38 |
thumbprint = "certificate's thumbprint" |
| 39 |
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_authentication(cluster, client_id, PEM, thumbprint, authority_id) |
| 40 |
|
| 41 |
# In case you want to authenticate with AAD application certificate Subject Name & Issuer |
| 42 |
filename = "path to a PEM certificate" |
| 43 |
with open(filename, "r") as pem_file: |
| 44 |
PEM = pem_file.read() |
| 45 |
|
| 46 |
filename = "path to a public certificate" |
| 47 |
with open(filename, "r") as cert_file: |
| 48 |
public_certificate = cert_file.read() |
| 49 |
|
| 50 |
thumbprint = "certificate's thumbprint" |
| 51 |
kcsb = KustoConnectionStringBuilder.with_aad_application_certificate_sni_authentication(cluster, client_id, PEM, public_certificate, thumbprint, authority_id) |
| 52 |
|
| 53 |
# In case you want to authenticate with a System Assigned Managed Service Identity (MSI) |
| 54 |
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster) |
| 55 |
|
| 56 |
# In case you want to authenticate with a User Assigned Managed Service Identity (MSI) |
| 57 |
user_assigned_client_id = "the AAD identity client id" |
| 58 |
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(cluster, client_id=user_assigned_client_id) |
| 59 |
|
| 60 |
# In case you want to authenticate with AAD username and password |
| 61 |
username = "<username>" |
| 62 |
password = "<password>" |
| 63 |
kcsb = KustoConnectionStringBuilder.with_aad_user_password_authentication(cluster, username, password, authority_id) |
| 64 |
|
| 65 |
# In case you want to authenticate with AAD device code. |
| 66 |
# Please note that if you choose this option, you'll need to autenticate for every new instance that is initialized. |
| 67 |
# It is highly recommended to create one instance and use it for all of your queries. |
| 68 |
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster) |
| 69 |
|
| 70 |
# The authentication method will be taken from the chosen KustoConnectionStringBuilder. |
| 71 |
client = QueuedIngestClient(kcsb) |
| 72 |
|
| 73 |
# there are more options for authenticating - see azure-kusto-data samples |
| 74 |
|
| 75 |
################################################################## |
| 76 |
## INGESTION ## |
| 77 |
################################################################## |
| 78 |
|
| 79 |
# there are a lot of useful properties, make sure to go over docs and check them out |
| 80 |
ingestion_props = IngestionProperties( |
| 81 |
database="{database_name}", |
| 82 |
table="{table_name}", |
| 83 |
data_format=DataFormat.CSV, |
| 84 |
# in case status update for success are also required (remember to import ReportLevel from azure.kusto.ingest) |
| 85 |
# report_level=ReportLevel.FailuresAndSuccesses, |
| 86 |
# in case a mapping is required (remember to import IngestionMappingKind from azure.kusto.data.data_format) |
| 87 |
# ingestion_mapping_reference="{json_mapping_that_already_exists_on_table}", |
| 88 |
# ingestion_mapping_kind= IngestionMappingKind.JSON, |
| 89 |
) |
| 90 |
|
| 91 |
# ingest from file |
| 92 |
file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes. |
| 93 |
client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_props) |
| 94 |
result = client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_props) |
| 95 |
|
| 96 |
# Inspect the result for useful information, such as source_id and blob_url |
| 97 |
print(repr(result)) |
| 98 |
|
| 99 |
# ingest from blob |
| 100 |
blob_descriptor = BlobDescriptor( |
| 101 |
"https://{path_to_blob}.csv.gz?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", |
| 102 |
10, |
| 103 |
) # 10 is the raw size of the data in bytes. |
| 104 |
client.ingest_from_blob(blob_descriptor, ingestion_properties=ingestion_props) |
| 105 |
|
| 106 |
# ingest from dataframe |
| 107 |
import pandas |
| 108 |
|
| 109 |
fields = ["id", "name", "value"] |
| 110 |
rows = [[1, "abc", 15.3], [2, "cde", 99.9]] |
| 111 |
|
| 112 |
df = pandas.DataFrame(data=rows, columns=fields) |
| 113 |
|
| 114 |
client.ingest_from_dataframe(df, ingestion_properties=ingestion_props) |
| 115 |
|
| 116 |
# ingest a whole folder. |
| 117 |
import os |
| 118 |
|
| 119 |
path = "folder/path" |
| 120 |
[client.ingest_from_file(f, ingestion_properties=ingestion_props) for f in os.listdir(path)] |
| 121 |
|
| 122 |
################################################################## |
| 123 |
## INGESTION STATUS ## |
| 124 |
################################################################## |
| 125 |
|
| 126 |
# if status updates are required, something like this can be done |
| 127 |
import pprint |
| 128 |
import time |
| 129 |
from azure.kusto.ingest.status import KustoIngestStatusQueues |
| 130 |
|
| 131 |
qs = KustoIngestStatusQueues(client) |
| 132 |
|
| 133 |
MAX_BACKOFF = 180 |
| 134 |
|
| 135 |
backoff = 1 |
| 136 |
while True: |
| 137 |
################### NOTICE #################### |
| 138 |
# in order to get success status updates, |
| 139 |
# make sure ingestion properties set the |
| 140 |
# reportLevel=ReportLevel.FailuresAndSuccesses. |
| 141 |
if qs.success.is_empty() and qs.failure.is_empty(): |
| 142 |
time.sleep(backoff) |
| 143 |
backoff = min(backoff * 2, MAX_BACKOFF) |
| 144 |
print("No new messages. backing off for {} seconds".format(backoff)) |
| 145 |
continue |
| 146 |
|
| 147 |
backoff = 1 |
| 148 |
|
| 149 |
success_messages = qs.success.pop(10) |
| 150 |
failure_messages = qs.failure.pop(10) |
| 151 |
|
| 152 |
pprint.pprint("SUCCESS : {}".format(success_messages)) |
| 153 |
pprint.pprint("FAILURE : {}".format(failure_messages)) |
| 154 |
|
| 155 |
# you can of course separate them and dump them into a file for follow up investigations |
| 156 |
with open("successes.log", "w+") as sf: |
| 157 |
for sm in success_messages: |
| 158 |
sf.write(str(sm)) |
| 159 |
|
| 160 |
with open("failures.log", "w+") as ff: |
| 161 |
for fm in failure_messages: |
| 162 |
ff.write(str(fm)) |
| 163 |
|
| 164 |
################################################################## |
| 165 |
## STREAMING INGEST ## |
| 166 |
################################################################## |
| 167 |
|
| 168 |
# Authenticate against this cluster endpoint as shows in the Auth section |
| 169 |
cluster = "https://{cluster_name}.kusto.windows.net" |
| 170 |
|
| 171 |
client = KustoStreamingIngestClient(kcsb) |
| 172 |
|
| 173 |
ingestion_properties = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV) |
| 174 |
|
| 175 |
# ingest from file |
| 176 |
file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes. |
| 177 |
client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties) |
| 178 |
client.ingest_from_file("{filename}.csv", ingestion_properties=ingestion_properties) |
| 179 |
|
| 180 |
# ingest from dataframe |
| 181 |
import pandas |
| 182 |
|
| 183 |
fields = ["id", "name", "value"] |
| 184 |
rows = [[1, "abc", 15.3], [2, "cde", 99.9]] |
| 185 |
|
| 186 |
df = pandas.DataFrame(data=rows, columns=fields) |
| 187 |
|
| 188 |
client.ingest_from_dataframe(df, ingestion_properties=ingestion_properties) |
| 189 |
|
| 190 |
# ingest from stream |
| 191 |
byte_sequence = b"56,56,56" |
| 192 |
bytes_stream = io.BytesIO(byte_sequence) |
| 193 |
client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) |
| 194 |
|
| 195 |
stream_descriptor = StreamDescriptor(bytes_stream) |
| 196 |
client.ingest_from_stream(stream_descriptor, ingestion_properties=ingestion_properties) |
| 197 |
|
| 198 |
str_sequence = "57,57,57" |
| 199 |
str_stream = io.StringIO(str_sequence) |
| 200 |
client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) |
| 201 |
|
| 202 |
################################################################## |
| 203 |
## NANAGED STREAMING INGEST ## |
| 204 |
################################################################## |
| 205 |
|
| 206 |
# Managed streaming ingest client will try to use streaming ingestion for performance, but will fall back to queued ingestion if unable. |
| 207 |
dm_cluster = "https://ingest-{cluster_name}.kusto.windows.net" |
| 208 |
|
| 209 |
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(dm_cluster, client_id, client_secret, authority_id) |
| 210 |
|
| 211 |
# Create it from a dm connection string |
| 212 |
client = ManagedStreamingIngestClient.from_dm_kcsb(kcsb) |
| 213 |
# or an engine connection string, like a streaming ingestion client with `from_engine_kcsb` |
| 214 |
# or provide both: `ManagedStreamingIngestClient(engine_kcsb, dm_kcsb)` |
| 215 |
|
| 216 |
# use client as you would a streaming or queued ingestion client |
| 217 |
|
| 218 |
byte_sequence = b"56,56,56" |
| 219 |
bytes_stream = io.BytesIO(byte_sequence) |
| 220 |
client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) |
| 221 |
|
| 222 |
ingestion_properties = IngestionProperties(database="{database_name}", table="{table_name}", data_format=DataFormat.CSV) |
| 223 |
|
| 224 |
# ingest from file |
| 225 |
file_descriptor = FileDescriptor("{filename}.csv", 3333) # 3333 is the raw size of the data in bytes. |
| 226 |
result = client.ingest_from_file(file_descriptor, ingestion_properties=ingestion_properties) |
| 227 |
|
| 228 |
# inspect the result to see what type of ingestion was preformed |
| 229 |
if result.status == IngestionStatus.QUEUED: |
| 230 |
# fell back to queued ingestion |
| 231 |
pass |
| 232 |
|
| 233 |
# Managed streaming ingest client will fall back to queued if: |
| 234 |
# - Multiple transient errors were encountered when trying to do streaming ingestion |
| 235 |
# - The ingestion is too large for streaming ingestion (over 4MB) |
| 236 |
# - The ingestion is directly for a blob |