Debian Codegraph @

azure-kusto-ingest/tests/test_kusto_streaming_ingest_client.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import io
4 import json
5 import os
6 import uuid
7
8 import pytest
9 import responses
10
11 from azure.kusto.data.data_format import DataFormat
12 from azure.kusto.ingest import KustoStreamingIngestClient, IngestionProperties, IngestionStatus, ManagedStreamingIngestClient
13
14 pandas_installed = False
15 try:
16 import pandas
17
18 pandas_installed = True
19 except:
20 pass
21
22 UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
23 BLOB_NAME_REGEX = "database__table__" + UUID_REGEX + "__dataset.csv.gz"
24 BLOB_URL_REGEX = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__" + UUID_REGEX + "__dataset.csv.gz[?]sas"
25
26
27 def request_callback(request, client_type, custom_request_id=None):
28 request_id = request.headers["x-ms-client-request-id"]
29 if custom_request_id:
30 assert custom_request_id == request_id
31 elif client_type == KustoStreamingIngestClient:
32 [prefix, request_uuid] = request_id.split(";")
33 assert prefix == "KPC.executeStreamingIngest"
34 uuid.UUID(request_uuid)
35 elif client_type == ManagedStreamingIngestClient:
36 assert_managed_streaming_request_id(request_id)
37
38 response_status = 200
39 response_headers = dict()
40 response_body = {
41 "Tables": [
42 {
43 "TableName": "Table_0",
44 "Columns": [
45 {"ColumnName": "ConsumedRecordsCount", "DataType": "Int64"},
46 {"ColumnName": "UpdatePolicyStatus", "DataType": "String"},
47 {"ColumnName": "UpdatePolicyFailureCode", "DataType": "String"},
48 {"ColumnName": "UpdatePolicyFailureReason", "DataType": "String"},
49 ],
50 "Rows": [[0, "Inactive", "Unknown", None]],
51 }
52 ]
53 }
54
55 return response_status, response_headers, json.dumps(response_body)
56
57
58 def assert_managed_streaming_request_id(request_id: str, retry: int = 0):
59 [prefix, request_uuid, suffix] = request_id.split(";")
60 assert prefix == "KPC.executeManagedStreamingIngest"
61 uuid.UUID(request_uuid)
62 assert int(suffix) == retry
63
64
65 @pytest.fixture(params=[KustoStreamingIngestClient, ManagedStreamingIngestClient])
66 def ingest_client_class(request):
67 if request.param == ManagedStreamingIngestClient:
68 return ManagedStreamingIngestClient
69 return request.param
70
71
72 class TestKustoStreamingIngestClient:
73 @responses.activate
74 def test_streaming_ingest_from_file(self, ingest_client_class):
75 responses.add_callback(
76 responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)
77 )
78
79 ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
80 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
81
82 # ensure test can work when executed from within directories
83 current_dir = os.getcwd()
84 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"]
85 missing_path_parts = []
86 for path_part in path_parts:
87 if path_part not in current_dir:
88 missing_path_parts.append(path_part)
89
90 file_path = os.path.join(current_dir, *missing_path_parts)
91
92 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
93 assert result.status == IngestionStatus.SUCCESS
94
95 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv.gz"]
96 missing_path_parts = []
97 for path_part in path_parts:
98 if path_part not in current_dir:
99 missing_path_parts.append(path_part)
100
101 file_path = os.path.join(current_dir, *missing_path_parts)
102
103 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
104 assert result.status == IngestionStatus.SUCCESS
105
106 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON, ingestion_mapping_reference="JsonMapping")
107
108 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.json"]
109 missing_path_parts = []
110 for path_part in path_parts:
111 if path_part not in current_dir:
112 missing_path_parts.append(path_part)
113
114 file_path = os.path.join(current_dir, *missing_path_parts)
115
116 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
117 assert result.status == IngestionStatus.SUCCESS
118
119 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.jsonz.gz"]
120 missing_path_parts = []
121 for path_part in path_parts:
122 if path_part not in current_dir:
123 missing_path_parts.append(path_part)
124
125 file_path = os.path.join(current_dir, *missing_path_parts)
126
127 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
128 assert result.status == IngestionStatus.SUCCESS
129
130 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.TSV)
131
132 path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.tsv"]
133 missing_path_parts = []
134 for path_part in path_parts:
135 if path_part not in current_dir:
136 missing_path_parts.append(path_part)
137
138 file_path = os.path.join(current_dir, *missing_path_parts)
139
140 result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties)
141 assert result.status == IngestionStatus.SUCCESS
142
143 @pytest.mark.skipif(not pandas_installed, reason="requires pandas")
144 @responses.activate
145 def test_streaming_ingest_from_dataframe(self, ingest_client_class):
146 responses.add_callback(
147 responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)
148 )
149
150 ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
151 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
152
153 from pandas import DataFrame
154
155 fields = ["id", "name", "value"]
156 rows = [[1, "abc", 15.3], [2, "cde", 99.9]]
157 df = DataFrame(data=rows, columns=fields)
158
159 result = ingest_client.ingest_from_dataframe(df, ingestion_properties)
160 assert result.status == IngestionStatus.SUCCESS
161
162 @responses.activate
163 def test_streaming_ingest_from_stream(self, ingest_client_class):
164 responses.add_callback(
165 responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class)
166 )
167
168 ingest_client = ingest_client_class("https://somecluster.kusto.windows.net")
169 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV)
170
171 byte_sequence = b"56,56,56"
172 bytes_stream = io.BytesIO(byte_sequence)
173 result = ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
174 assert result.status == IngestionStatus.SUCCESS
175
176 str_sequence = "57,57,57"
177 str_stream = io.StringIO(str_sequence)
178 result = ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
179 assert result.status == IngestionStatus.SUCCESS
180
181 byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}'
182 bytes_stream = io.BytesIO(byte_sequence)
183
184 # Make sure we don't raise when not providing a mapping to JSON
185 ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON)
186
187 ingestion_properties.ingestion_mapping_reference = "JsonMapping"
188 result = ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties)
189 assert result.status == IngestionStatus.SUCCESS
190
191 str_sequence = '{"Name":"Ben","Age":"56","Weight":"75"}'
192 str_stream = io.StringIO(str_sequence)
193 result = ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties)
194 assert result.status == IngestionStatus.SUCCESS
195
196 def test_client_uri_from_query_endpoint(self):
197 assert (
198 KustoStreamingIngestClient("https://somecluster.kusto.windows.net")._kusto_client._kusto_cluster == "https://somecluster.kusto.windows.net/"
199 ), "Client URI was not extracted correctly from query endpoint"
200
201 def test_client_uri_from_ingestion_endpoint(self):
202 assert (
203 KustoStreamingIngestClient("https://ingest-somecluster.kusto.windows.net")._kusto_client._kusto_cluster == "https://somecluster.kusto.windows.net/"
204 ), "Client URI was not extracted correctly from ingestion endpoint"