| 1 |
from typing import Any, Tuple, Dict, Iterator |
| 2 |
|
| 3 |
import aiohttp |
| 4 |
import ijson |
| 5 |
from ijson import IncompleteJSONError |
| 6 |
|
| 7 |
from azure.kusto.data._models import WellKnownDataSet |
| 8 |
from azure.kusto.data.exceptions import KustoTokenParsingError, KustoUnsupportedApiError, KustoApiError, KustoMultiApiError |
| 9 |
from azure.kusto.data.streaming_response import JsonTokenType, FrameType, JsonToken |
| 10 |
|
| 11 |
|
| 12 |
class JsonTokenReader: |
| 13 |
def __init__(self, stream: aiohttp.StreamReader): |
| 14 |
self.json_iter = ijson.parse_async(stream, use_float=True) |
| 15 |
|
| 16 |
def __aiter__(self) -> "JsonTokenReader": |
| 17 |
return self |
| 18 |
|
| 19 |
def __anext__(self) -> JsonToken: |
| 20 |
return self.read_next_token_or_throw() |
| 21 |
|
| 22 |
async def read_next_token_or_throw(self) -> JsonToken: |
| 23 |
try: |
| 24 |
next_item = await self.json_iter.__anext__() |
| 25 |
except IncompleteJSONError: |
| 26 |
next_item = None |
| 27 |
if next_item is None: |
| 28 |
raise KustoTokenParsingError("Unexpected end of stream") |
| 29 |
(token_path, token_type, token_value) = next_item |
| 30 |
|
| 31 |
return JsonToken(token_path, JsonTokenType[token_type.upper()], token_value) |
| 32 |
|
| 33 |
async def read_token_of_type(self, *token_types: JsonTokenType) -> JsonToken: |
| 34 |
token = await self.read_next_token_or_throw() |
| 35 |
if token.token_type not in token_types: |
| 36 |
raise KustoTokenParsingError(f"Expected one the following types: '{','.join(t.name for t in token_types)}' , got type {token.token_type}") |
| 37 |
return token |
| 38 |
|
| 39 |
async def read_start_object(self) -> JsonToken: |
| 40 |
return await self.read_token_of_type(JsonTokenType.START_MAP) |
| 41 |
|
| 42 |
async def read_start_array(self) -> JsonToken: |
| 43 |
return await self.read_token_of_type(JsonTokenType.START_ARRAY) |
| 44 |
|
| 45 |
async def read_string(self) -> str: |
| 46 |
return (await self.read_token_of_type(JsonTokenType.STRING)).token_value |
| 47 |
|
| 48 |
async def read_boolean(self) -> bool: |
| 49 |
return (await self.read_token_of_type(JsonTokenType.BOOLEAN)).token_value |
| 50 |
|
| 51 |
async def read_number(self) -> float: |
| 52 |
return (await self.read_token_of_type(JsonTokenType.NUMBER)).token_value |
| 53 |
|
| 54 |
async def skip_children(self, prev_token: JsonToken): |
| 55 |
if prev_token.token_type == JsonTokenType.MAP_KEY: |
| 56 |
prev_token = await self.read_next_token_or_throw() |
| 57 |
if prev_token.token_type in JsonTokenType.start_tokens(): |
| 58 |
async for potential_end_token in self: |
| 59 |
if potential_end_token.token_path == prev_token.token_path and potential_end_token.token_type in JsonTokenType.end_tokens(): |
| 60 |
break |
| 61 |
|
| 62 |
async def skip_until_property_name(self, name: str) -> JsonToken: |
| 63 |
while True: |
| 64 |
token = await self.read_token_of_type(JsonTokenType.MAP_KEY) |
| 65 |
if token.token_value == name: |
| 66 |
return token |
| 67 |
|
| 68 |
await self.skip_children(token) |
| 69 |
|
| 70 |
async def skip_until_any_property_name(self, *names: str) -> JsonToken: |
| 71 |
while True: |
| 72 |
token = await self.read_token_of_type(JsonTokenType.MAP_KEY) |
| 73 |
if token.token_value in names: |
| 74 |
return token |
| 75 |
|
| 76 |
await self.skip_children(token) |
| 77 |
|
| 78 |
async def skip_until_property_name_or_end_object(self, *names: str) -> JsonToken: |
| 79 |
async for token in self: |
| 80 |
if token.token_type == JsonTokenType.END_MAP: |
| 81 |
return token |
| 82 |
|
| 83 |
if token.token_type == JsonTokenType.MAP_KEY: |
| 84 |
if token.token_value in names: |
| 85 |
return token |
| 86 |
|
| 87 |
await self.skip_children(token) |
| 88 |
continue |
| 89 |
|
| 90 |
raise Exception(f"Unexpected token {token}") |
| 91 |
|
| 92 |
async def skip_until_token_with_paths(self, *tokens: (JsonTokenType, str)) -> JsonToken: |
| 93 |
async for token in self: |
| 94 |
if any((token.token_type == t_type and token.token_path == t_path) for (t_type, t_path) in tokens): |
| 95 |
return token |
| 96 |
await self.skip_children(token) |
| 97 |
|
| 98 |
|
| 99 |
class StreamingDataSetEnumerator: |
| 100 |
def __init__(self, reader: JsonTokenReader): |
| 101 |
self.reader = reader |
| 102 |
self.done = False |
| 103 |
self.started = False |
| 104 |
self.started_primary_results = False |
| 105 |
self.finished_primary_results = False |
| 106 |
|
| 107 |
def __aiter__(self) -> "StreamingDataSetEnumerator": |
| 108 |
return self |
| 109 |
|
| 110 |
async def __anext__(self) -> Dict[str, Any]: |
| 111 |
if self.done: |
| 112 |
raise StopIteration() |
| 113 |
|
| 114 |
if not self.started: |
| 115 |
await self.reader.read_start_array() |
| 116 |
self.started = True |
| 117 |
|
| 118 |
token = await self.reader.skip_until_token_with_paths((JsonTokenType.START_MAP, "item"), (JsonTokenType.END_ARRAY, "")) |
| 119 |
if token == JsonTokenType.END_ARRAY: |
| 120 |
self.done = True |
| 121 |
raise StopIteration() |
| 122 |
|
| 123 |
frame_type = await self.read_frame_type() |
| 124 |
parsed_frame = await self.parse_frame(frame_type) |
| 125 |
is_primary_result = parsed_frame["FrameType"] == FrameType.DataTable and parsed_frame["TableKind"] == WellKnownDataSet.PrimaryResult.value |
| 126 |
if is_primary_result: |
| 127 |
self.started_primary_results = True |
| 128 |
elif self.started_primary_results: |
| 129 |
self.finished_primary_results = True |
| 130 |
|
| 131 |
return parsed_frame |
| 132 |
|
| 133 |
async def parse_frame(self, frame_type: FrameType) -> Dict[str, Any]: |
| 134 |
if frame_type == FrameType.DataSetHeader: |
| 135 |
frame = await self.extract_props(frame_type, ("IsProgressive", JsonTokenType.BOOLEAN), ("Version", JsonTokenType.STRING)) |
| 136 |
if frame["IsProgressive"]: |
| 137 |
raise KustoUnsupportedApiError.progressive_api_unsupported() |
| 138 |
return frame |
| 139 |
if frame_type in [FrameType.TableHeader, FrameType.TableFragment, FrameType.TableCompletion, FrameType.TableProgress]: |
| 140 |
raise KustoUnsupportedApiError.progressive_api_unsupported() |
| 141 |
if frame_type == FrameType.DataTable: |
| 142 |
props = await self.extract_props( |
| 143 |
frame_type, |
| 144 |
("TableId", JsonTokenType.NUMBER), |
| 145 |
("TableKind", JsonTokenType.STRING), |
| 146 |
("TableName", JsonTokenType.STRING), |
| 147 |
("Columns", JsonTokenType.START_ARRAY), |
| 148 |
) |
| 149 |
await self.reader.skip_until_property_name("Rows") |
| 150 |
props["Rows"] = self.row_iterator() |
| 151 |
if props["TableKind"] != WellKnownDataSet.PrimaryResult.value: |
| 152 |
props["Rows"] = [r async for r in props["Rows"]] |
| 153 |
return props |
| 154 |
if frame_type == FrameType.DataSetCompletion: |
| 155 |
res = await self.extract_props(frame_type, ("HasErrors", JsonTokenType.BOOLEAN), ("Cancelled", JsonTokenType.BOOLEAN)) |
| 156 |
token = await self.reader.skip_until_property_name_or_end_object("OneApiErrors") |
| 157 |
if token.token_type != JsonTokenType.END_MAP: |
| 158 |
res["OneApiErrors"] = self.parse_array(skip_start=False) |
| 159 |
return res |
| 160 |
|
| 161 |
async def row_iterator(self) -> Iterator[list]: |
| 162 |
await self.reader.read_token_of_type(JsonTokenType.START_ARRAY) |
| 163 |
while True: |
| 164 |
token = await self.reader.read_token_of_type(JsonTokenType.START_ARRAY, JsonTokenType.END_ARRAY, JsonTokenType.START_MAP) |
| 165 |
if token.token_type == JsonTokenType.START_MAP: |
| 166 |
raise KustoMultiApiError([await self.parse_object(skip_start=True)]) |
| 167 |
if token.token_type == JsonTokenType.END_ARRAY: |
| 168 |
return |
| 169 |
yield await self.parse_array(skip_start=True) |
| 170 |
|
| 171 |
async def parse_array(self, skip_start: bool) -> list: |
| 172 |
if not skip_start: |
| 173 |
await self.reader.read_start_array() |
| 174 |
arr = [] |
| 175 |
|
| 176 |
while True: |
| 177 |
token = await self.reader.read_token_of_type( |
| 178 |
JsonTokenType.NULL, |
| 179 |
JsonTokenType.BOOLEAN, |
| 180 |
JsonTokenType.NUMBER, |
| 181 |
JsonTokenType.STRING, |
| 182 |
JsonTokenType.START_MAP, |
| 183 |
JsonTokenType.START_ARRAY, |
| 184 |
JsonTokenType.END_ARRAY, |
| 185 |
) |
| 186 |
|
| 187 |
if token.token_type == JsonTokenType.END_ARRAY: |
| 188 |
return arr |
| 189 |
|
| 190 |
if token.token_type == JsonTokenType.START_MAP: |
| 191 |
arr.append(await self.parse_object(skip_start=True)) |
| 192 |
elif token.token_type == JsonTokenType.START_ARRAY: |
| 193 |
arr.append(await self.parse_array(skip_start=True)) |
| 194 |
else: |
| 195 |
arr.append(token.token_value) |
| 196 |
|
| 197 |
async def parse_object(self, skip_start: bool) -> Dict[str, Any]: |
| 198 |
if not skip_start: |
| 199 |
await self.reader.read_start_object() |
| 200 |
|
| 201 |
obj = {} |
| 202 |
while True: |
| 203 |
token_prop_name = await self.reader.read_token_of_type(JsonTokenType.MAP_KEY, JsonTokenType.END_MAP) |
| 204 |
if token_prop_name.token_type == JsonTokenType.END_MAP: |
| 205 |
return obj |
| 206 |
prop_name = token_prop_name.token_value |
| 207 |
|
| 208 |
token = await self.reader.read_token_of_type( |
| 209 |
JsonTokenType.NULL, JsonTokenType.BOOLEAN, JsonTokenType.NUMBER, JsonTokenType.STRING, JsonTokenType.START_MAP, JsonTokenType.START_ARRAY |
| 210 |
) |
| 211 |
|
| 212 |
if token.token_type == JsonTokenType.START_MAP: |
| 213 |
obj[prop_name] = await self.parse_object(skip_start=True) |
| 214 |
elif token.token_type == JsonTokenType.START_ARRAY: |
| 215 |
obj[prop_name] = await self.parse_array(skip_start=True) |
| 216 |
else: |
| 217 |
obj[prop_name] = token.token_value |
| 218 |
|
| 219 |
async def extract_props(self, frame_type: FrameType, *props: Tuple[str, JsonTokenType]) -> Dict[str, Any]: |
| 220 |
result = {"FrameType": frame_type} |
| 221 |
props_dict = dict(props) |
| 222 |
while props_dict: |
| 223 |
name = (await self.reader.skip_until_any_property_name(*props_dict.keys())).token_value |
| 224 |
if props_dict[name] == JsonTokenType.START_ARRAY: |
| 225 |
result[name] = await self.parse_array(skip_start=False) |
| 226 |
else: |
| 227 |
result[name] = (await self.reader.read_token_of_type(props_dict[name])).token_value |
| 228 |
props_dict.pop(name) |
| 229 |
|
| 230 |
return result |
| 231 |
|
| 232 |
async def read_frame_type(self) -> FrameType: |
| 233 |
await self.reader.skip_until_property_name("FrameType") |
| 234 |
return FrameType[await self.reader.read_string()] |