Debian Codegraph @

azure-kusto-ingest/tests/test_kusto_ingest_client.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import io
4 import json
5 import os
6 import uuid
7 from pathlib import Path
8 from unittest.mock import patch
9
10 import pytest
11 import responses
12
13 from azure.kusto.data.data_format import DataFormat
14
15 from azure.kusto.ingest import QueuedIngestClient, IngestionProperties, IngestionStatus, _resource_manager
16 from azure.kusto.ingest.exceptions import KustoInvalidEndpointError, KustoQueueError
17 from azure.kusto.ingest.managed_streaming_ingest_client import ManagedStreamingIngestClient
18
19 pandas_installed = False
20 try:
21 import pandas
22
23 pandas_installed = True
24 except:
25 pass
26
27 UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
28 BLOB_NAME_REGEX = "database__table__" + UUID_REGEX + "__dataset.csv.gz"
29 BLOB_URL_REGEX = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__" + UUID_REGEX + "__dataset.csv.gz[?]sas"
30 STORAGE_QUEUE_URL = "https://storageaccount.queue.core.windows.net/readyforaggregation-secured?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
31 TEMP_STORAGE_URL = "https://storageaccount.blob.core.windows.net/tempstorage?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
32 TEMP_STORAGE2_URL = "https://storageaccount2.blob.core.windows.net/tempstorage2?sp=2"
33 TEMP_STORAGE3_URL = "https://storageaccount2.blob.core.windows.net/tempstorage2?sp=3"
34 TEMP_STORAGE4_URL = "https://storageaccount2.blob.core.windows.net/tempstorage2?sp=4"
35 TEMP_STORAGE5_URL = "https://storageaccount2.blob.core.windows.net/tempstorage2?sp=5"
36 TEMP_STORAGE6_URL = "https://storageaccount3.blob.core.windows.net/tempstorage3?sp=6"
37 TEMP_STORAGE7_URL = "https://storageaccount3.blob.core.windows.net/tempstorage3?sp=7"
38 TEMP_STORAGE8_URL = "https://storageaccount3.blob.core.windows.net/tempstorage3?sp=8"
39 TEMP_STORAGE9_URL = "https://storageaccount3.blob.core.windows.net/tempstorage4?sp=9"
40
41 STORAGE_QUEUE2_URL = "https://storageaccount3.queue.core.windows.net/readyforaggregation-secured?2"
42 STORAGE_QUEUE3_URL = "https://storageaccount3.queue.core.windows.net/readyforaggregation-secured?3"
43 STORAGE_QUEUE4_URL = "https://storageaccount2.queue.core.windows.net/readyforaggregation-secured?4"
44 STORAGE_QUEUE5_URL = "https://storageaccount2.queue.core.windows.net/readyforaggregation-secured?5"
45
46
47 def request_callback_throw_transient(request):
48 response_status = 400
49 response_body = {
50 "error": {
51 "code": "BadRequest",
52 "message": "Request is invalid and cannot be executed.",
53 "@type": "Kusto.Common.Svc.Exceptions.AdminCommandWrongEndpointException",
54 "@message": "Cannot get ingestion resources from this service endpoint. The appropriate endpoint is most likely "
55 "'https://ingest-somecluster.kusto.windows.net/'.",
56 "@context": {
57 "timestamp": "2021-10-12T06:05:35.6602087Z",
58 "serviceAlias": "SomeCluster",
59 "machineName": "KEngine000000",
60 "processName": "Kusto.WinSvc.Svc",
61 "processId": 2648,
62 "threadId": 472,
63 "appDomainName": "Kusto.WinSvc.Svc.exe",
64 "clientRequestId": "KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674",
65 "activityId": "87eb8fc9-78b3-4580-bcc8-6c90482f9118",
66 "subActivityId": "bbfb038b-4467-4f96-afd4-945904fc6278",
67 "activityType": "DN.AdminCommand.IngestionResourcesGetCommand",
68 "parentActivityId": "00e678e9-4204-4143-8c94-6afd94c27430",
69 "activityStack": "(Activity stack: CRID=KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674 ARID=87eb8fc9-78b3-4580-bcc8-6c90482f9118 > DN.Admin.Client.ExecuteControlCommand/833dfb85-5d67-44b7-882d-eb2283e65780 > P.WCF.Service.ExecuteControlCommand..IInterNodeCommunicationAdminContract/3784e74f-1d89-4c15-adef-0a360c4c431e > DN.FE.ExecuteControlCommand/00e678e9-4204-4143-8c94-6afd94c27430 > DN.AdminCommand.IngestionResourcesGetCommand/bbfb038b-4467-4f96-afd4-945904fc6278)",
70 },
71 "@permanent": False,
72 }
73 }
74 return response_status, {}, json.dumps(response_body)
75
76
77 def request_callback(request):
78 body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
79 response_status = 400
80 response_headers = dict()
81 response_body = {}
82
83 if ".get ingestion resources" in body["csl"]:
84 response_status = 200
85 response_body = {
86 "Tables": [
87 {
88 "TableName": "Table_0",
89 "Columns": [{"ColumnName": "ResourceTypeName", "DataType": "String"}, {"ColumnName": "StorageRoot", "DataType": "String"}],
90 "Rows": [
91 [
92 "SecuredReadyForAggregationQueue",
93 STORAGE_QUEUE_URL,
94 ],
95 [
96 "SecuredReadyForAggregationQueue",
97 STORAGE_QUEUE_URL,
98 ],
99 [
100 "SecuredReadyForAggregationQueue",
101 STORAGE_QUEUE_URL,
102 ],
103 [
104 "SecuredReadyForAggregationQueue",
105 STORAGE_QUEUE_URL,
106 ],
107 [
108 "SecuredReadyForAggregationQueue",
109 STORAGE_QUEUE_URL,
110 ],
111 [
112 "FailedIngestionsQueue",
113 "https://storageaccount.queue.core.windows.net/failedingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
114 ],
115 [
116 "SuccessfulIngestionsQueue",
117 "https://storageaccount.queue.core.windows.net/successfulingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
118 ],
119 [
120 "TempStorage",
121 TEMP_STORAGE_URL,
122 ],
123 [
124 "TempStorage",
125 TEMP_STORAGE_URL,
126 ],
127 [
128 "TempStorage",
129 TEMP_STORAGE_URL,
130 ],
131 [
132 "TempStorage",
133 TEMP_STORAGE_URL,
134 ],
135 [
136 "TempStorage",
137 TEMP_STORAGE_URL,
138 ],
139 [
140 "IngestionsStatusTable",
141 "https://storageaccount.table.core.windows.net/ingestionsstatus?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
142 ],
143 ],
144 }
145 ]
146 }
147
148 if ".get kusto identity token" in body["csl"]:
149 response_status = 200
150 response_body = {
151 "Tables": [{"TableName": "Table_0", "Columns": [{"ColumnName": "AuthorizationContext", "DataType": "String"}], "Rows": [["authorization_context"]]}]
152 }
153
154 return response_status, response_headers, json.dumps(response_body)
155
156
157 def request_callback_check_retries(request):
158 body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
159 response_status = 400
160 response_headers = dict()
161 response_body = {}
162
163 if ".get ingestion resources" in body["csl"]:
164 response_status = 200
165 response_body = {
166 "Tables": [
167 {
168 "TableName": "Table_0",
169 "Columns": [{"ColumnName": "ResourceTypeName", "DataType": "String"}, {"ColumnName": "StorageRoot", "DataType": "String"}],
170 "Rows": [
171 [
172 "SecuredReadyForAggregationQueue",
173 STORAGE_QUEUE2_URL,
174 ],
175 [
176 "SecuredReadyForAggregationQueue",
177 STORAGE_QUEUE3_URL,
178 ],
179 [
180 "SecuredReadyForAggregationQueue",
181 STORAGE_QUEUE_URL,
182 ],
183 [
184 "SecuredReadyForAggregationQueue",
185 STORAGE_QUEUE4_URL,
186 ],
187 [
188 "SecuredReadyForAggregationQueue",
189 STORAGE_QUEUE5_URL,
190 ],
191 [
192 "FailedIngestionsQueue",
193 "https://storageaccount.queue.core.windows.net/failedingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
194 ],
195 [
196 "SuccessfulIngestionsQueue",
197 "https://storageaccount.queue.core.windows.net/successfulingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
198 ],
199 [
200 "TempStorage",
201 TEMP_STORAGE_URL,
202 ],
203 [
204 "TempStorage",
205 TEMP_STORAGE2_URL,
206 ],
207 [
208 "TempStorage",
209 TEMP_STORAGE3_URL,
210 ],
211 [
212 "TempStorage",
213 TEMP_STORAGE4_URL,
214 ],
215 [
216 "TempStorage",
217 TEMP_STORAGE5_URL,
218 ],
219 [
220 "TempStorage",
221 TEMP_STORAGE6_URL,
222 ],
223 [
224 "IngestionsStatusTable",
225 "https://storageaccount.table.core.windows.net/ingestionsstatus?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
226 ],
227 ],
228 }
229 ]
230 }
231
232 if ".get kusto identity token" in body["csl"]:
233 response_status = 200
234 response_body = {
235 "Tables": [{"TableName": "Table_0", "Columns": [{"ColumnName": "AuthorizationContext", "DataType": "String"}], "Rows": [["authorization_context"]]}]
236 }
237
238 return response_status, response_headers, json.dumps(response_body)
239
240
241 def request_callback_all_retries_failed(request):
242 body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
243 response_status = 400
244 response_headers = dict()
245 response_body = {}
246
247 if ".get ingestion resources" in body["csl"]:
248 response_status = 200
249 response_body = {
250 "Tables": [
251 {
252 "TableName": "Table_0",
253 "Columns": [{"ColumnName": "ResourceTypeName", "DataType": "String"}, {"ColumnName": "StorageRoot", "DataType": "String"}],
254 "Rows": [
255 [
256 "SecuredReadyForAggregationQueue",
257 STORAGE_QUEUE2_URL,
258 ],
259 [
260 "FailedIngestionsQueue",
261 "https://storageaccount.queue.core.windows.net/failedingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
262 ],
263 [
264 "SuccessfulIngestionsQueue",
265 "https://storageaccount.queue.core.windows.net/successfulingestions?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
266 ],
267 [
268 "TempStorage",
269 TEMP_STORAGE6_URL,
270 ],
271 [
272 "IngestionsStatusTable",
273 "https://storageaccount.table.core.windows.net/ingestionsstatus?sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
274 ],
275 ],
276 }
277 ]
278 }
279
280 if ".get kusto identity token" in body["csl"]:
281 response_status = 200
282 response_body = {
283 "Tables": [{"TableName": "Table_0", "Columns": [{"ColumnName": "AuthorizationContext", "DataType": "String"}], "Rows": [["authorization_context"]]}]
284 }
285
286 return response_status, response_headers, json.dumps(response_body)
287
288
289 def request_callback_containers(request):
290 body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
291 response_headers = dict()
292
293 if ".get ingestion resources" in body["csl"]:
294 response_status = 200
295 response_body = {
296 "Tables": [
297 {
298 "TableName": "Table_0",
299 "Columns": [{"ColumnName": "ResourceTypeName", "DataType": "String"}, {"ColumnName": "StorageRoot", "DataType": "String"}],
300 "Rows": [
301 [
302 "TempStorage",
303 TEMP_STORAGE2_URL,
304 ],
305 [
306 "TempStorage",
307 TEMP_STORAGE3_URL,
308 ],
309 [
310 "TempStorage",
311 TEMP_STORAGE6_URL,
312 ],
313 [
314 "TempStorage",
315 TEMP_STORAGE7_URL,
316 ],
317 ],
318 }
319 ]
320 }
321
322 return response_status, response_headers, json.dumps(response_body)
323
324
325 def request_error_callback(request):
326 body = json.loads(request.body.decode()) if type(request.body) == bytes else json.loads(request.body)
327 response_status = 400
328 response_headers = dict()
329 response_body = {}
330
331 if ".get ingestion resources" in body["csl"]:
332 response_status = 400
333 response_body = {
334 "error": {
335 "code": "BadRequest",
336 "message": "Request is invalid and cannot be executed.",
337 "@type": "Kusto.Common.Svc.Exceptions.AdminCommandWrongEndpointException",
338 "@message": "Cannot get ingestion resources from this service endpoint. The appropriate endpoint is most likely "
339 "'https://ingest-somecluster.kusto.windows.net/'.",
340 "@context": {
341 "timestamp": "2021-10-12T06:05:35.6602087Z",
342 "serviceAlias": "SomeCluster",
343 "machineName": "KEngine000000",
344 "processName": "Kusto.WinSvc.Svc",
345 "processId": 2648,
346 "threadId": 472,
347 "appDomainName": "Kusto.WinSvc.Svc.exe",
348 "clientRequestId": "KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674",
349 "activityId": "87eb8fc9-78b3-4580-bcc8-6c90482f9118",
350 "subActivityId": "bbfb038b-4467-4f96-afd4-945904fc6278",
351 "activityType": "DN.AdminCommand.IngestionResourcesGetCommand",
352 "parentActivityId": "00e678e9-4204-4143-8c94-6afd94c27430",
353 "activityStack": "(Activity stack: CRID=KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674 ARID=87eb8fc9-78b3-4580-bcc8-6c90482f9118 > DN.Admin.Client.ExecuteControlCommand/833dfb85-5d67-44b7-882d-eb2283e65780 > P.WCF.Service.ExecuteControlCommand..IInterNodeCommunicationAdminContract/3784e74f-1d89-4c15-adef-0a360c4c431e > DN.FE.ExecuteControlCommand/00e678e9-4204-4143-8c94-6afd94c27430 > DN.AdminCommand.IngestionResourcesGetCommand/bbfb038b-4467-4f96-afd4-945904fc6278)",
354 },
355 "@permanent": True,
356 }
357 }
358
359 if ".show version" in body["csl"]:
360 response_status = 200
361 response_body = {
362 "Tables": [
363 {
364 "TableName": "Table_0",
365 "Columns": [
366 {"ColumnName": "BuildVersion", "DataType": "String"},
367 {"ColumnName": "BuildTime", "DataType": "DateTime"},
368 {"ColumnName": "ServiceType", "DataType": "String"},
369 {"ColumnName": "ProductVersion", "DataType": "String"},
370 ],
371 "Rows": [["1.0.0.0", "2000-01-01T00:00:00Z", "Engine", "2020-09-07 12-09-22"]],
372 }
373 ]
374 }
375
376 return response_status, response_headers, json.dumps(response_body)
377
378
379 def assert_queued_upload(mock_put_message_in_queue, mock_upload_blob_from_stream, expected_url: str, check_raw_data: bool = True, format: str = "csv"):
380 # mock_put_message_in_queue
381 assert mock_put_message_in_queue.call_count == 1
382
383 put_message_in_queue_mock_kwargs = mock_put_message_in_queue.call_args_list[0][1]
384
385 queued_message_json = json.loads(put_message_in_queue_mock_kwargs["content"])
386
387 # mock_upload_blob_from_stream
388 # not checking the query string because it can change order, just checking it's there
389 assert queued_message_json["BlobPath"].startswith(expected_url) is True
390 assert len(queued_message_json["BlobPath"]) > len(expected_url)
391 assert queued_message_json["DatabaseName"] == "database"
392 assert queued_message_json["IgnoreSizeLimit"] is False
393 assert queued_message_json["AdditionalProperties"]["format"] == format
394 assert queued_message_json["FlushImmediately"] is False
395 assert queued_message_json["TableName"] == "table"
396 if check_raw_data:
397 assert queued_message_json["RawDataSize"] > 0
398 assert queued_message_json["RetainBlobOnSuccess"] is True
399 if mock_upload_blob_from_stream is not None:
400 upload_blob_kwargs = mock_upload_blob_from_stream.call_args_list[0][1]
401 assert issubclass(type(upload_blob_kwargs["data"]), io.BufferedIOBase)
402
403
404 @pytest.fixture(params=[QueuedIngestClient, ManagedStreamingIngestClient])
405 def ingest_client_class(request):
406 if request.param == ManagedStreamingIngestClient:
407 return ManagedStreamingIngestClient
408 return request.param
409
410
411 class TestQueuedIngestClient:
412 MOCKED_UUID_4 = uuid.UUID("11111111-1111-1111-1111-111111111111")
413 MOCKED_PID = 64
414 MOCKED_TIME = 100
415
416 @responses.activate
417 @patch(
418 "azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0
419 ) # Trick to always fallback to queued ingest
420 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
421 @patch("azure.storage.blob.BlobClient.upload_blob")
422 @patch("azure.storage.queue.QueueClient.send_message")
423 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
424 def test_sanity_ingest_from_file(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, ingest_client_class):
425 responses.add_callback(
426 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback, content_type="application/json"
427 )
428
429 ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
430 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
431
432 # ensure test can work when executed from within directories
433 current_dir = os.getcwd()
434 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
435 missing_path_parts = []
436 for path_part in path_parts:
437 if path_part not in current_dir:
438 missing_path_parts.append(path_part)
439
440 file_path = os.path.join(current_dir, *missing_path_parts)
441
442 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
443
444 assert result.status == IngestionStatus.QUEUED
445
446 assert_queued_upload(
447 mock_put_message_in_queue,
448 mock_upload_blob_from_stream,
449 "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__dataset.csv.gz?",
450 )
451
452 ingest_client.close()
453
454 @responses.activate
455 @patch(
456 "azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0
457 ) # Trick to always fallback to queued ingest
458 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
459 @patch("azure.storage.blob.BlobClient.upload_blob")
460 @patch("azure.storage.queue.QueueClient.send_message")
461 def test_sanity_ingest_from_file_when_different_storage_accounts(
462 self, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, ingest_client_class
463 ):
464 responses.add_callback(
465 responses.POST,
466 "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt",
467 callback=request_callback_check_retries,
468 content_type="application/json",
469 )
470
471 ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
472 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
473
474 # ensure test can work when executed from within directories
475 current_dir = os.getcwd()
476 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
477 missing_path_parts = []
478 for path_part in path_parts:
479 if path_part not in current_dir:
480 missing_path_parts.append(path_part)
481
482 file_path = os.path.join(current_dir, *missing_path_parts)
483
484 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
485
486 assert result.status == IngestionStatus.QUEUED
487
488 assert_queued_upload(
489 mock_put_message_in_queue,
490 mock_upload_blob_from_stream,
491 "https://storageaccount",
492 )
493
494 ingest_client.close()
495
496 @responses.activate
497 @pytest.mark.skipif(not pandas_installed, reason="requires pandas")
498 @patch("azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0)
499 @patch("azure.storage.blob.BlobClient.upload_blob")
500 @patch("azure.storage.queue.QueueClient.send_message")
501 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
502 @patch("time.time", return_value=MOCKED_TIME)
503 @patch("os.getpid", return_value=MOCKED_PID)
504 def test_simple_ingest_from_dataframe(self, mock_pid, mock_time, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, ingest_client_class):
505 responses.add_callback(
506 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback, content_type="application/json"
507 )
508
509 ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
510 ingestion_properties = IngestionProperties(database="database", table="table")
511
512 from pandas import DataFrame
513
514 fields = ["id", "name", "value"]
515 rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
516 df = DataFrame(data=rows, columns=fields)
517
518 result = ingest_client.ingest_from_dataframe(df, ingestion_properties=ingestion_properties)
519 assert result.status == IngestionStatus.QUEUED
520
521 expected_url = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__df_{0}_100_11111111-1111-1111-1111-111111111111.json.gz?".format(
522 id(df)
523 )
524
525 assert_queued_upload(mock_put_message_in_queue, mock_upload_blob_from_stream, expected_url, format="json")
526
527 @responses.activate
528 @patch("azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0)
529 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
530 @patch("azure.storage.blob.BlobClient.upload_blob")
531 @patch("azure.storage.queue.QueueClient.send_message")
532 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
533 def test_sanity_ingest_from_stream(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, ingest_client_class):
534 responses.add_callback(
535 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback, content_type="application/json"
536 )
537
538 ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
539 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
540
541 # ensure test can work when executed from within directories
542 current_dir = os.getcwd()
543 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
544 missing_path_parts = []
545
546 for path_part in path_parts:
547 if path_part not in current_dir:
548 missing_path_parts.append(path_part)
549
550 file_path = os.path.join(current_dir, *missing_path_parts)
551
552 result = ingest_client.ingest_from_stream(io.StringIO(Path(file_path).read_text()), ingestion_properties=ingestion_properties)
553 assert result.status == IngestionStatus.QUEUED
554
555 assert_queued_upload(
556 mock_put_message_in_queue,
557 mock_upload_blob_from_stream,
558 "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__stream.gz?",
559 check_raw_data=False,
560 )
561
562 @responses.activate
563 def test_containers(self):
564 responses.add_callback(
565 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=request_callback_containers, content_type="application/json"
566 )
567
568 kustoClient = _resource_manager.KustoClient("https://ingest-somecluster.kusto.windows.net")
569 ResourceManager = _resource_manager._ResourceManager(kustoClient)
570
571 containers_selected_with_round_robin = ResourceManager.get_containers()
572
573 # Verify correct number of containers
574 assert len(containers_selected_with_round_robin) == 4
575 # Verify correct distribution of containers
576 assert containers_selected_with_round_robin[0].storage_account_name == containers_selected_with_round_robin[2].storage_account_name
577 assert containers_selected_with_round_robin[1].storage_account_name == containers_selected_with_round_robin[3].storage_account_name
578 assert containers_selected_with_round_robin[1].storage_account_name != containers_selected_with_round_robin[2].storage_account_name
579
580 @responses.activate
581 @patch(
582 "azure.kusto.ingest.managed_streaming_ingest_client.ManagedStreamingIngestClient.MAX_STREAMING_SIZE_IN_BYTES", new=0
583 ) # Trick to always fallback to queued ingest
584 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
585 @patch("azure.storage.blob.BlobClient.upload_blob")
586 def test_queue_all_retries_failed(self, mock_upload_blob_from_stream, mock_aad, ingest_client_class):
587 responses.add_callback(
588 responses.POST,
589 "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt",
590 callback=request_callback_all_retries_failed,
591 content_type="application/json",
592 )
593
594 ingest_client = ingest_client_class("https://ingest-somecluster.kusto.windows.net")
595 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
596
597 # ensure test can work when executed from within directories
598 current_dir = os.getcwd()
599 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
600 missing_path_parts = []
601 for path_part in path_parts:
602 if path_part not in current_dir:
603 missing_path_parts.append(path_part)
604
605 file_path = os.path.join(current_dir, *missing_path_parts)
606 with pytest.raises(KustoQueueError):
607 ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
608
609 ingest_client.close()
610
611 def test_client_uri_from_query_endpoint(self):
612 client = QueuedIngestClient("https://somecluster.kusto.windows.net")
613 assert client._connection_datasource == "https://ingest-somecluster.kusto.windows.net", "Client URI was not extracted correctly from query endpoint"
614
615 assert client._resource_manager._kusto_client._kusto_cluster == "https://ingest-somecluster.kusto.windows.net/"
616
617 def test_client_uri_from_ingestion_endpoint(self):
618 client = QueuedIngestClient("https://ingest-somecluster.kusto.windows.net")
619 assert client._connection_datasource == "https://ingest-somecluster.kusto.windows.net", "Client URI was not extracted correctly from query endpoint"
620
621 assert client._resource_manager._kusto_client._kusto_cluster == "https://ingest-somecluster.kusto.windows.net/"
622
623 def test_from_ingestion_endpoint_reserved_hostname(self):
624 client = QueuedIngestClient("https://localhost:8080")
625 assert client._connection_datasource == "https://localhost:8080", "Client URI was not extracted correctly from query endpoint"
626
627 assert client._resource_manager._kusto_client._kusto_cluster == "https://localhost:8080/"
628
629 client = QueuedIngestClient("https://192.168.14.4")
630 assert client._connection_datasource == "https://192.168.14.4", "Client URI was not extracted correctly from query endpoint"
631
632 assert client._resource_manager._kusto_client._kusto_cluster == "https://192.168.14.4/"
633
634 client = QueuedIngestClient("https://onebox.dev.kusto.windows.net")
635 assert client._connection_datasource == "https://onebox.dev.kusto.windows.net", "Client URI was not extracted correctly from query endpoint"
636
637 assert client._resource_manager._kusto_client._kusto_cluster == "https://onebox.dev.kusto.windows.net/"