Debian Codegraph @

azure-kusto-ingest/tests/test_managed_streaming_ingest.py

raw
1 import io
2 import json
3 import os
4 import uuid
5 from tempfile import NamedTemporaryFile
6 from unittest.mock import patch
7
8 import pytest
9 import responses
10
11 from azure.kusto.data.data_format import DataFormat
12 from azure.kusto.data.exceptions import KustoApiError
13 from azure.kusto.ingest import ManagedStreamingIngestClient, IngestionProperties, IngestionStatus, BlobDescriptor
14 from test_kusto_ingest_client import request_callback as queued_request_callback, assert_queued_upload, request_callback_throw_transient
15 from test_kusto_streaming_ingest_client import request_callback as streaming_request_callback, assert_managed_streaming_request_id
16
17
18 class TransientResponseHelper:
19 def __init__(self, times_to_fail):
20 self.times_to_fail = times_to_fail
21 self.total_calls = 0
22
23
24 @pytest.fixture(params=["Blob", "File"])
25 def is_blob(request):
26 return request.param == "Blob"
27
28
29 def transient_error_callback(helper: TransientResponseHelper, request, custom_request_id=None):
30 if custom_request_id:
31 assert request.headers["x-ms-client-request-id"] == custom_request_id
32 else:
33 assert_managed_streaming_request_id(request.headers["x-ms-client-request-id"], helper.total_calls)
34
35 response_headers = dict()
36 helper.total_calls += 1
37
38 if helper.total_calls <= helper.times_to_fail:
39 response_status = 400
40 response_body = {
41 "error": {
42 "code": "General_InternalServerError",
43 "message": "Unknown error",
44 "@type": "Kusto.DataNode.Exceptions.InternalServerError",
45 "@message": "InternalServerError",
46 "@context": {
47 "timestamp": "2021-10-21T14:12:00.7878847Z",
48 "serviceAlias": "SomeCluster",
49 "machineName": "KEngine000000",
50 "processName": "Kusto.WinSvc.Svc",
51 "processId": 3040,
52 "threadId": 7524,
53 "appDomainName": "Kusto.WinSvc.Svc.exe",
54 "clientRequestId": "KNC.executeStreamingIngest;a4d80ca5-8729-404b-b745-0d5555737483",
55 "activityId": "bace3d9d-d949-457e-b741-1d2c6bc56658",
56 "subActivityId": "bace3d9d-d949-457e-b741-1d2c6bc56658",
57 "activityType": "PO.OWIN.CallContext",
58 "parentActivityId": "bace3d9d-d949-457e-b741-1d2c6bc56658",
59 "activityStack": "(Activity stack: CRID=KNC.executeStreamingIngest;a4d80ca5-8729-404b-b745-0d5555737493 ARID=bace3d9d-d949-457e-b741-1d2c6bc56678 > PO.OWIN.CallContext/bace3d9d-d949-457e-b741-1d2c6bc56678 > PO.OWIN.CallContext/bace3d9d-d949-457e-b741-1d2c6bc56678)",
60 },
61 "@permanent": False,
62 }
63 }
64 else:
65 response_status = 200
66 response_body = {
67 "Tables": [
68 {
69 "TableName": "Table_0",
70 "Columns": [
71 {"ColumnName": "ConsumedRecordsCount", "DataType": "Int64"},
72 {"ColumnName": "UpdatePolicyStatus", "DataType": "String"},
73 {"ColumnName": "UpdatePolicyFailureCode", "DataType": "String"},
74 {"ColumnName": "UpdatePolicyFailureReason", "DataType": "String"},
75 ],
76 "Rows": [[0, "Inactive", "Unknown", None]],
77 }
78 ]
79 }
80
81 return response_status, response_headers, json.dumps(response_body)
82
83
84 class TestManagedStreamingIngestClient:
85 MOCKED_UUID_4 = uuid.UUID("11111111-1111-1111-1111-111111111111")
86
87 @responses.activate
88 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
89 @patch("azure.storage.blob.BlobClient.upload_blob")
90 @patch("azure.storage.queue.QueueClient.send_message")
91 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
92 def test_fallback_big_file(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad, is_blob):
93 responses.add_callback(
94 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
95 )
96 responses.add_callback(
97 responses.POST,
98 "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
99 callback=streaming_request_callback,
100 content_type="application/json",
101 )
102
103 data_format = DataFormat.ORC # Using orc to avoid compression
104 ingest_client = ManagedStreamingIngestClient("https://ingest-somecluster.kusto.windows.net")
105 ingestion_properties = IngestionProperties(database="database", table="table", data_format=data_format)
106
107 initial_bytes = bytearray(os.urandom(5 * 1024 * 1024))
108
109 def check_bytes(data, **kwargs):
110 assert kwargs["timeout"] == 10 * 60
111 assert data.read() == initial_bytes
112
113 mock_upload_blob_from_stream.side_effect = check_bytes
114
115 f = NamedTemporaryFile(dir=".", mode="wb", delete=False)
116 blob_url = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__{}?".format(
117 os.path.basename(f.name)
118 )
119
120 try:
121 if is_blob:
122 result = ingest_client.ingest_from_blob(BlobDescriptor(blob_url + "sas", 5 * 1024 * 1024), ingestion_properties=ingestion_properties)
123 f.close()
124 else:
125 f.write(initial_bytes)
126 f.close()
127 result = ingest_client.ingest_from_file(f.name, ingestion_properties=ingestion_properties)
128 except Exception as e:
129 print(e)
130 finally:
131 os.unlink(f.name)
132
133 assert result.status == IngestionStatus.QUEUED
134
135 assert_queued_upload(
136 mock_put_message_in_queue,
137 mock_upload_blob_from_stream if not is_blob else None,
138 blob_url,
139 format=data_format.kusto_value,
140 )
141
142 if not is_blob:
143 mock_upload_blob_from_stream.assert_called()
144
145 @responses.activate
146 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
147 @patch("azure.storage.blob.BlobClient.upload_blob")
148 @patch("azure.storage.queue.QueueClient.send_message")
149 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
150 def test_fallback_big_stream(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad):
151 responses.add_callback(
152 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
153 )
154 responses.add_callback(
155 responses.POST,
156 "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
157 callback=streaming_request_callback,
158 content_type="application/json",
159 )
160
161 data_format = DataFormat.ORC # Using orc to avoid compression
162 ingest_client = ManagedStreamingIngestClient("https://somecluster.kusto.windows.net")
163 ingestion_properties = IngestionProperties(database="database", table="table", data_format=data_format)
164
165 initial_bytes = bytearray(os.urandom(5 * 1024 * 1024))
166 stream = io.BytesIO(initial_bytes)
167
168 def check_bytes(data, **kwargs):
169 assert kwargs["timeout"] == 10 * 60
170 assert data.read() == initial_bytes
171
172 mock_upload_blob_from_stream.side_effect = check_bytes
173
174 result = ingest_client.ingest_from_stream(stream, ingestion_properties=ingestion_properties)
175
176 assert result.status == IngestionStatus.QUEUED
177
178 assert_queued_upload(
179 mock_put_message_in_queue,
180 mock_upload_blob_from_stream,
181 "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__stream?",
182 format=data_format.kusto_value,
183 check_raw_data=False,
184 )
185
186 mock_upload_blob_from_stream.assert_called()
187
188 @responses.activate
189 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
190 @patch("azure.storage.blob.BlobClient.upload_blob")
191 @patch("azure.storage.queue.QueueClient.send_message")
192 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
193 def test_fallback_transient_errors_limit(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad):
194 total_attempts = 3
195
196 responses.add_callback(
197 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
198 )
199
200 ingest_client = ManagedStreamingIngestClient("https://somecluster.kusto.windows.net")
201 ingest_client._set_retry_settings(0)
202 ingestion_properties = IngestionProperties(database="database", table="table")
203
204 helper = TransientResponseHelper(times_to_fail=total_attempts)
205 responses.add_callback(
206 responses.POST,
207 "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
208 callback=lambda request: transient_error_callback(helper, request),
209 content_type="application/json",
210 )
211
212 # ensure test can work when executed from within directories
213 current_dir = os.getcwd()
214 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
215 missing_path_parts = []
216 for path_part in path_parts:
217 if path_part not in current_dir:
218 missing_path_parts.append(path_part)
219
220 file_path = os.path.join(current_dir, *missing_path_parts)
221
222 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
223
224 assert result.status == IngestionStatus.QUEUED
225
226 assert_queued_upload(
227 mock_put_message_in_queue,
228 mock_upload_blob_from_stream,
229 "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__dataset.csv.gz?",
230 )
231
232 assert helper.total_calls == total_attempts
233
234 @responses.activate
235 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
236 @patch("azure.storage.blob.BlobClient.upload_blob")
237 @patch("azure.storage.queue.QueueClient.send_message")
238 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
239 def test_fallback_transient_single_error(self, mock_uuid, mock_put_message_in_queue, mock_upload_blob_from_stream, mock_aad):
240 total_failures = 2
241
242 responses.add_callback(
243 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
244 )
245 helper = TransientResponseHelper(times_to_fail=total_failures)
246 responses.add_callback(
247 responses.POST,
248 "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
249 callback=lambda request: transient_error_callback(helper, request),
250 content_type="application/json",
251 )
252
253 ingest_client = ManagedStreamingIngestClient("https://somecluster.kusto.windows.net")
254 ingest_client._set_retry_settings(0)
255 ingestion_properties = IngestionProperties(database="database", table="table")
256
257 # ensure test can work when executed from within directories
258 current_dir = os.getcwd()
259 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
260 missing_path_parts = []
261 for path_part in path_parts:
262 if path_part not in current_dir:
263 missing_path_parts.append(path_part)
264
265 file_path = os.path.join(current_dir, *missing_path_parts)
266
267 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
268
269 assert result.status == IngestionStatus.SUCCESS
270
271 assert helper.total_calls == total_failures + 1
272
273 @responses.activate
274 def test_permanent_error(self):
275 responses.add(
276 responses.POST,
277 "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table",
278 status=400,
279 json={
280 "error": {
281 "code": "BadRequest",
282 "message": "Request is invalid and cannot be executed.",
283 "@type": "Kusto.Common.Svc.Exceptions.AdminCommandWrongEndpointException",
284 "@message": "Cannot get ingestion resources from this service endpoint. The appropriate endpoint is most likely "
285 "'https://ingest-somecluster.kusto.windows.net/'.",
286 "@context": {
287 "timestamp": "2021-10-12T06:05:35.6602087Z",
288 "serviceAlias": "SomeCluster",
289 "machineName": "KEngine000000",
290 "processName": "Kusto.WinSvc.Svc",
291 "processId": 2648,
292 "threadId": 472,
293 "appDomainName": "Kusto.WinSvc.Svc.exe",
294 "clientRequestId": "KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674",
295 "activityId": "87eb8fc9-78b3-4580-bcc8-6c90482f9118",
296 "subActivityId": "bbfb038b-4467-4f96-afd4-945904fc6278",
297 "activityType": "DN.AdminCommand.IngestionResourcesGetCommand",
298 "parentActivityId": "00e678e9-4204-4143-8c94-6afd94c27430",
299 "activityStack": "(Activity stack: CRID=KPC.execute;a3dfb878-9d2b-49d6-89a5-e9b3a9f1f674 ARID=87eb8fc9-78b3-4580-bcc8-6c90482f9118 > DN.Admin.Client.ExecuteControlCommand/833dfb85-5d67-44b7-882d-eb2283e65780 > P.WCF.Service.ExecuteControlCommand..IInterNodeCommunicationAdminContract/3784e74f-1d89-4c15-adef-0a360c4c431e > DN.FE.ExecuteControlCommand/00e678e9-4204-4143-8c94-6afd94c27430 > DN.AdminCommand.IngestionResourcesGetCommand/bbfb038b-4467-4f96-afd4-945904fc6278)",
300 },
301 "@permanent": True,
302 }
303 },
304 content_type="application/json",
305 )
306
307 ingest_client = ManagedStreamingIngestClient("https://ingest-somecluster.kusto.windows.net")
308 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
309
310 current_dir = os.getcwd()
311 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
312 missing_path_parts = []
313 for path_part in path_parts:
314 if path_part not in current_dir:
315 missing_path_parts.append(path_part)
316
317 file_path = os.path.join(current_dir, *missing_path_parts)
318
319 with pytest.raises(KustoApiError) as ex:
320 ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
321 assert ex.value.get_api_error().permanent == True
322
323 @responses.activate
324 @patch("azure.kusto.data.security._AadHelper.acquire_authorization_header", return_value=None)
325 @patch("azure.storage.queue.QueueClient.send_message")
326 @patch("uuid.uuid4", return_value=MOCKED_UUID_4)
327 def test_blob_ingestion(self, mock_uuid, mock_put_message_in_queue, mock_aad):
328 responses.add_callback(
329 responses.POST, "https://ingest-somecluster.kusto.windows.net/v1/rest/mgmt", callback=queued_request_callback, content_type="application/json"
330 )
331 responses.add_callback(
332 responses.POST,
333 "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table?streamFormat=csv&sourceKind=uri",
334 callback=request_callback_throw_transient,
335 content_type="application/json",
336 )
337
338 ingest_client = ManagedStreamingIngestClient("https://ingest-somecluster.kusto.windows.net")
339 ingestion_properties = IngestionProperties(database="database", table="table")
340
341 blob_path = (
342 "https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__tmpbvk40leg?sp=rl&st=2020-05-20T13"
343 "%3A38%3A37Z&se=2020-05-21T13%3A38%3A37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx "
344 )
345 result = ingest_client.ingest_from_blob(BlobDescriptor(blob_path, 1), ingestion_properties=ingestion_properties)
346
347 assert result.status == IngestionStatus.QUEUED
348
349 assert_queued_upload(
350 mock_put_message_in_queue,
351 mock_upload_blob_from_stream=None,
352 expected_url="https://storageaccount.blob.core.windows.net/tempstorage/database__table__11111111-1111-1111-1111-111111111111__tmpbvk40leg?",
353 )
354
355 def test_client_uri_from_query_endpoint(self):
356 client = ManagedStreamingIngestClient("https://somecluster.kusto.windows.net")
357 assert (
358 client.queued_client._connection_datasource == "https://ingest-somecluster.kusto.windows.net"
359 ), "Client URI was not extracted correctly from query endpoint"
360
361 assert client.queued_client._resource_manager._kusto_client._kusto_cluster == "https://ingest-somecluster.kusto.windows.net/"
362
363 assert (
364 client.streaming_client._kusto_client._kusto_cluster == "https://somecluster.kusto.windows.net/"
365 ), "Client URI was not extracted correctly from ingestion endpoint"
366
367 def test_client_uri_from_ingestion_endpoint(self):
368 client = ManagedStreamingIngestClient("https://ingest-somecluster.kusto.windows.net")
369 assert (
370 client.queued_client._connection_datasource == "https://ingest-somecluster.kusto.windows.net"
371 ), "Client URI was not extracted correctly from query endpoint"
372
373 assert client.queued_client._resource_manager._kusto_client._kusto_cluster == "https://ingest-somecluster.kusto.windows.net/"
374
375 assert (
376 client.streaming_client._kusto_client._kusto_cluster == "https://somecluster.kusto.windows.net/"
377 ), "Client URI was not extracted correctly from ingestion endpoint"
378
379 def test_from_ingestion_endpoint_reserved_hostname(self):
380 client = ManagedStreamingIngestClient("https://localhost:8080")
381 assert client.queued_client._connection_datasource == "https://localhost:8080", "Client URI was not extracted correctly from query endpoint"
382 assert client.queued_client._resource_manager._kusto_client._kusto_cluster == "https://localhost:8080/"
383 assert client.streaming_client._kusto_client._kusto_cluster == "https://localhost:8080/"
384
385 client = ManagedStreamingIngestClient("https://192.168.14.4")
386 assert client.queued_client._connection_datasource == "https://192.168.14.4", "Client URI was not extracted correctly from query endpoint"
387 assert client.queued_client._resource_manager._kusto_client._kusto_cluster == "https://192.168.14.4/"
388 assert client.streaming_client._kusto_client._kusto_cluster == "https://192.168.14.4/"
389
390 client = ManagedStreamingIngestClient("https://onebox.dev.kusto.windows.net")
391 assert (
392 client.queued_client._connection_datasource == "https://onebox.dev.kusto.windows.net"
393 ), "Client URI was not extracted correctly from query endpoint"
394 assert client.queued_client._resource_manager._kusto_client._kusto_cluster == "https://onebox.dev.kusto.windows.net/"
395 assert client.streaming_client._kusto_client._kusto_cluster == "https://onebox.dev.kusto.windows.net/"