| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
import io |
| 4 |
import json |
| 5 |
import os |
| 6 |
import uuid |
| 7 |
|
| 8 |
import pytest |
| 9 |
import responses |
| 10 |
|
| 11 |
from azure.kusto.data.data_format import DataFormat |
| 12 |
from azure.kusto.ingest import KustoStreamingIngestClient, IngestionProperties, IngestionStatus, ManagedStreamingIngestClient |
| 13 |
|
| 14 |
pandas_installed = False |
| 15 |
try: |
| 16 |
import pandas |
| 17 |
|
| 18 |
pandas_installed = True |
| 19 |
except: |
| 20 |
pass |
| 21 |
|
| 22 |
UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}" |
| 23 |
BLOB_NAME_REGEX = "database__table__" + UUID_REGEX + "__dataset.csv.gz" |
| 24 |
BLOB_URL_REGEX = "https://storageaccount.blob.core.windows.net/tempstorage/database__table__" + UUID_REGEX + "__dataset.csv.gz[?]sas" |
| 25 |
|
| 26 |
|
| 27 |
def request_callback(request, client_type, custom_request_id=None): |
| 28 |
request_id = request.headers["x-ms-client-request-id"] |
| 29 |
if custom_request_id: |
| 30 |
assert custom_request_id == request_id |
| 31 |
elif client_type == KustoStreamingIngestClient: |
| 32 |
[prefix, request_uuid] = request_id.split(";") |
| 33 |
assert prefix == "KPC.executeStreamingIngest" |
| 34 |
uuid.UUID(request_uuid) |
| 35 |
elif client_type == ManagedStreamingIngestClient: |
| 36 |
assert_managed_streaming_request_id(request_id) |
| 37 |
|
| 38 |
response_status = 200 |
| 39 |
response_headers = dict() |
| 40 |
response_body = { |
| 41 |
"Tables": [ |
| 42 |
{ |
| 43 |
"TableName": "Table_0", |
| 44 |
"Columns": [ |
| 45 |
{"ColumnName": "ConsumedRecordsCount", "DataType": "Int64"}, |
| 46 |
{"ColumnName": "UpdatePolicyStatus", "DataType": "String"}, |
| 47 |
{"ColumnName": "UpdatePolicyFailureCode", "DataType": "String"}, |
| 48 |
{"ColumnName": "UpdatePolicyFailureReason", "DataType": "String"}, |
| 49 |
], |
| 50 |
"Rows": [[0, "Inactive", "Unknown", None]], |
| 51 |
} |
| 52 |
] |
| 53 |
} |
| 54 |
|
| 55 |
return response_status, response_headers, json.dumps(response_body) |
| 56 |
|
| 57 |
|
| 58 |
def assert_managed_streaming_request_id(request_id: str, retry: int = 0): |
| 59 |
[prefix, request_uuid, suffix] = request_id.split(";") |
| 60 |
assert prefix == "KPC.executeManagedStreamingIngest" |
| 61 |
uuid.UUID(request_uuid) |
| 62 |
assert int(suffix) == retry |
| 63 |
|
| 64 |
|
| 65 |
@pytest.fixture(params=[KustoStreamingIngestClient, ManagedStreamingIngestClient]) |
| 66 |
def ingest_client_class(request): |
| 67 |
if request.param == ManagedStreamingIngestClient: |
| 68 |
return ManagedStreamingIngestClient |
| 69 |
return request.param |
| 70 |
|
| 71 |
|
| 72 |
class TestKustoStreamingIngestClient: |
| 73 |
@responses.activate |
| 74 |
def test_streaming_ingest_from_file(self, ingest_client_class): |
| 75 |
responses.add_callback( |
| 76 |
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class) |
| 77 |
) |
| 78 |
|
| 79 |
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net") |
| 80 |
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV) |
| 81 |
|
| 82 |
# ensure test can work when executed from within directories |
| 83 |
current_dir = os.getcwd() |
| 84 |
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv"] |
| 85 |
missing_path_parts = [] |
| 86 |
for path_part in path_parts: |
| 87 |
if path_part not in current_dir: |
| 88 |
missing_path_parts.append(path_part) |
| 89 |
|
| 90 |
file_path = os.path.join(current_dir, *missing_path_parts) |
| 91 |
|
| 92 |
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties) |
| 93 |
assert result.status == IngestionStatus.SUCCESS |
| 94 |
|
| 95 |
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.csv.gz"] |
| 96 |
missing_path_parts = [] |
| 97 |
for path_part in path_parts: |
| 98 |
if path_part not in current_dir: |
| 99 |
missing_path_parts.append(path_part) |
| 100 |
|
| 101 |
file_path = os.path.join(current_dir, *missing_path_parts) |
| 102 |
|
| 103 |
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties) |
| 104 |
assert result.status == IngestionStatus.SUCCESS |
| 105 |
|
| 106 |
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON, ingestion_mapping_reference="JsonMapping") |
| 107 |
|
| 108 |
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.json"] |
| 109 |
missing_path_parts = [] |
| 110 |
for path_part in path_parts: |
| 111 |
if path_part not in current_dir: |
| 112 |
missing_path_parts.append(path_part) |
| 113 |
|
| 114 |
file_path = os.path.join(current_dir, *missing_path_parts) |
| 115 |
|
| 116 |
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties) |
| 117 |
assert result.status == IngestionStatus.SUCCESS |
| 118 |
|
| 119 |
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.jsonz.gz"] |
| 120 |
missing_path_parts = [] |
| 121 |
for path_part in path_parts: |
| 122 |
if path_part not in current_dir: |
| 123 |
missing_path_parts.append(path_part) |
| 124 |
|
| 125 |
file_path = os.path.join(current_dir, *missing_path_parts) |
| 126 |
|
| 127 |
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties) |
| 128 |
assert result.status == IngestionStatus.SUCCESS |
| 129 |
|
| 130 |
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.TSV) |
| 131 |
|
| 132 |
path_parts = ["azure-kusto-ingest", "tests", "input", "dataset.tsv"] |
| 133 |
missing_path_parts = [] |
| 134 |
for path_part in path_parts: |
| 135 |
if path_part not in current_dir: |
| 136 |
missing_path_parts.append(path_part) |
| 137 |
|
| 138 |
file_path = os.path.join(current_dir, *missing_path_parts) |
| 139 |
|
| 140 |
result = ingest_client.ingest_from_file(file_path, ingestion_properties=ingestion_properties) |
| 141 |
assert result.status == IngestionStatus.SUCCESS |
| 142 |
|
| 143 |
@pytest.mark.skipif(not pandas_installed, reason="requires pandas") |
| 144 |
@responses.activate |
| 145 |
def test_streaming_ingest_from_dataframe(self, ingest_client_class): |
| 146 |
responses.add_callback( |
| 147 |
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class) |
| 148 |
) |
| 149 |
|
| 150 |
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net") |
| 151 |
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV) |
| 152 |
|
| 153 |
from pandas import DataFrame |
| 154 |
|
| 155 |
fields = ["id", "name", "value"] |
| 156 |
rows = [[1, "abc", 15.3], [2, "cde", 99.9]] |
| 157 |
df = DataFrame(data=rows, columns=fields) |
| 158 |
|
| 159 |
result = ingest_client.ingest_from_dataframe(df, ingestion_properties) |
| 160 |
assert result.status == IngestionStatus.SUCCESS |
| 161 |
|
| 162 |
@responses.activate |
| 163 |
def test_streaming_ingest_from_stream(self, ingest_client_class): |
| 164 |
responses.add_callback( |
| 165 |
responses.POST, "https://somecluster.kusto.windows.net/v1/rest/ingest/database/table", callback=lambda r: request_callback(r, ingest_client_class) |
| 166 |
) |
| 167 |
|
| 168 |
ingest_client = ingest_client_class("https://somecluster.kusto.windows.net") |
| 169 |
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.CSV) |
| 170 |
|
| 171 |
byte_sequence = b"56,56,56" |
| 172 |
bytes_stream = io.BytesIO(byte_sequence) |
| 173 |
result = ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) |
| 174 |
assert result.status == IngestionStatus.SUCCESS |
| 175 |
|
| 176 |
str_sequence = "57,57,57" |
| 177 |
str_stream = io.StringIO(str_sequence) |
| 178 |
result = ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) |
| 179 |
assert result.status == IngestionStatus.SUCCESS |
| 180 |
|
| 181 |
byte_sequence = b'{"Name":"Ben","Age":"56","Weight":"75"}' |
| 182 |
bytes_stream = io.BytesIO(byte_sequence) |
| 183 |
|
| 184 |
# Make sure we don't raise when not providing a mapping to JSON |
| 185 |
ingestion_properties = IngestionProperties(database="database", table="table", data_format=DataFormat.JSON) |
| 186 |
|
| 187 |
ingestion_properties.ingestion_mapping_reference = "JsonMapping" |
| 188 |
result = ingest_client.ingest_from_stream(bytes_stream, ingestion_properties=ingestion_properties) |
| 189 |
assert result.status == IngestionStatus.SUCCESS |
| 190 |
|
| 191 |
str_sequence = '{"Name":"Ben","Age":"56","Weight":"75"}' |
| 192 |
str_stream = io.StringIO(str_sequence) |
| 193 |
result = ingest_client.ingest_from_stream(str_stream, ingestion_properties=ingestion_properties) |
| 194 |
assert result.status == IngestionStatus.SUCCESS |
| 195 |
|
| 196 |
def test_client_uri_from_query_endpoint(self): |
| 197 |
assert ( |
| 198 |
KustoStreamingIngestClient("https://somecluster.kusto.windows.net")._kusto_client._kusto_cluster == "https://somecluster.kusto.windows.net/" |
| 199 |
), "Client URI was not extracted correctly from query endpoint" |
| 200 |
|
| 201 |
def test_client_uri_from_ingestion_endpoint(self): |
| 202 |
assert ( |
| 203 |
KustoStreamingIngestClient("https://ingest-somecluster.kusto.windows.net")._kusto_client._kusto_cluster == "https://somecluster.kusto.windows.net/" |
| 204 |
), "Client URI was not extracted correctly from ingestion endpoint" |