| 1 |
from enum import Enum |
| 2 |
from typing import Optional, Any, Tuple, Dict, AnyStr, IO, List, Iterator |
| 3 |
|
| 4 |
import ijson |
| 5 |
from ijson import IncompleteJSONError |
| 6 |
|
| 7 |
from azure.kusto.data._models import WellKnownDataSet |
| 8 |
from azure.kusto.data.exceptions import KustoServiceError, KustoTokenParsingError, KustoUnsupportedApiError, KustoApiError, KustoMultiApiError |
| 9 |
|
| 10 |
|
| 11 |
class JsonTokenType(Enum): |
| 12 |
NULL = 0 |
| 13 |
BOOLEAN = 1 |
| 14 |
NUMBER = 2 |
| 15 |
STRING = 3 |
| 16 |
MAP_KEY = 4 |
| 17 |
START_MAP = 5 |
| 18 |
END_MAP = 6 |
| 19 |
START_ARRAY = 7 |
| 20 |
END_ARRAY = 8 |
| 21 |
|
| 22 |
@staticmethod |
| 23 |
def start_tokens() -> "List[JsonTokenType]": |
| 24 |
return [JsonTokenType.START_MAP, JsonTokenType.START_ARRAY] |
| 25 |
|
| 26 |
@staticmethod |
| 27 |
def end_tokens() -> "List[JsonTokenType]": |
| 28 |
return [JsonTokenType.END_MAP, JsonTokenType.END_ARRAY] |
| 29 |
|
| 30 |
|
| 31 |
class FrameType(Enum): |
| 32 |
DataSetHeader = 0 |
| 33 |
TableHeader = 1 |
| 34 |
TableFragment = 2 |
| 35 |
TableCompletion = 3 |
| 36 |
TableProgress = 4 |
| 37 |
DataTable = 5 |
| 38 |
DataSetCompletion = 6 |
| 39 |
|
| 40 |
|
| 41 |
class JsonToken: |
| 42 |
def __init__(self, token_path: str, token_type: JsonTokenType, token_value: Optional[Any]): |
| 43 |
self.token_path = token_path |
| 44 |
self.token_type = token_type |
| 45 |
self.token_value = token_value |
| 46 |
|
| 47 |
|
| 48 |
class JsonTokenReader: |
| 49 |
def __init__(self, stream: IO[AnyStr]): |
| 50 |
self.json_iter = ijson.parse(stream, use_float=True) |
| 51 |
|
| 52 |
def __iter__(self) -> "JsonTokenReader": |
| 53 |
return self |
| 54 |
|
| 55 |
def __next__(self) -> JsonToken: |
| 56 |
return self.read_next_token_or_throw() |
| 57 |
|
| 58 |
def read_next_token_or_throw(self) -> JsonToken: |
| 59 |
try: |
| 60 |
next_item = next(self.json_iter) |
| 61 |
except IncompleteJSONError: |
| 62 |
next_item = None |
| 63 |
if next_item is None: |
| 64 |
raise KustoTokenParsingError("Unexpected end of stream") |
| 65 |
(token_path, token_type, token_value) = next_item |
| 66 |
|
| 67 |
return JsonToken(token_path, JsonTokenType[token_type.upper()], token_value) |
| 68 |
|
| 69 |
def read_token_of_type(self, *token_types: JsonTokenType) -> JsonToken: |
| 70 |
token = self.read_next_token_or_throw() |
| 71 |
if token.token_type not in token_types: |
| 72 |
raise KustoTokenParsingError(f"Expected one the following types: '{','.join(t.name for t in token_types)}' , got type {token.token_type}") |
| 73 |
return token |
| 74 |
|
| 75 |
def read_start_object(self) -> JsonToken: |
| 76 |
return self.read_token_of_type(JsonTokenType.START_MAP) |
| 77 |
|
| 78 |
def read_start_array(self) -> JsonToken: |
| 79 |
return self.read_token_of_type(JsonTokenType.START_ARRAY) |
| 80 |
|
| 81 |
def read_string(self) -> str: |
| 82 |
return self.read_token_of_type(JsonTokenType.STRING).token_value |
| 83 |
|
| 84 |
def read_boolean(self) -> bool: |
| 85 |
return self.read_token_of_type(JsonTokenType.BOOLEAN).token_value |
| 86 |
|
| 87 |
def read_number(self) -> float: |
| 88 |
return self.read_token_of_type(JsonTokenType.NUMBER).token_value |
| 89 |
|
| 90 |
def skip_children(self, prev_token: JsonToken): |
| 91 |
if prev_token.token_type == JsonTokenType.MAP_KEY: |
| 92 |
prev_token = self.read_next_token_or_throw() |
| 93 |
|
| 94 |
if prev_token.token_type in JsonTokenType.start_tokens(): |
| 95 |
for potential_end_token in self: |
| 96 |
if potential_end_token.token_path == prev_token.token_path and potential_end_token.token_type in JsonTokenType.end_tokens(): |
| 97 |
break |
| 98 |
|
| 99 |
def skip_until_property_name(self, name: str) -> JsonToken: |
| 100 |
while True: |
| 101 |
token = self.read_token_of_type(JsonTokenType.MAP_KEY) |
| 102 |
if token.token_value == name: |
| 103 |
return token |
| 104 |
|
| 105 |
self.skip_children(token) |
| 106 |
|
| 107 |
def skip_until_any_property_name(self, *names: str) -> JsonToken: |
| 108 |
while True: |
| 109 |
token = self.read_token_of_type(JsonTokenType.MAP_KEY) |
| 110 |
if token.token_value in names: |
| 111 |
return token |
| 112 |
|
| 113 |
self.skip_children(token) |
| 114 |
|
| 115 |
def skip_until_property_name_or_end_object(self, *names: str) -> JsonToken: |
| 116 |
for token in self: |
| 117 |
if token.token_type == JsonTokenType.END_MAP: |
| 118 |
return token |
| 119 |
|
| 120 |
if token.token_type == JsonTokenType.MAP_KEY: |
| 121 |
if token.token_value in names: |
| 122 |
return token |
| 123 |
|
| 124 |
self.skip_children(token) |
| 125 |
continue |
| 126 |
|
| 127 |
raise Exception(f"Unexpected token {token}") |
| 128 |
|
| 129 |
def skip_until_token_with_paths(self, *tokens: (JsonTokenType, str)) -> JsonToken: |
| 130 |
for token in self: |
| 131 |
if any((token.token_type == t_type and token.token_path == t_path) for (t_type, t_path) in tokens): |
| 132 |
return token |
| 133 |
self.skip_children(token) |
| 134 |
|
| 135 |
|
| 136 |
class StreamingDataSetEnumerator: |
| 137 |
def __init__(self, reader: JsonTokenReader): |
| 138 |
self.reader = reader |
| 139 |
self.done = False |
| 140 |
self.started = False |
| 141 |
self.started_primary_results = False |
| 142 |
self.finished_primary_results = False |
| 143 |
|
| 144 |
def __iter__(self) -> "StreamingDataSetEnumerator": |
| 145 |
return self |
| 146 |
|
| 147 |
def __next__(self) -> Dict[str, Any]: |
| 148 |
if self.done: |
| 149 |
raise StopIteration() |
| 150 |
|
| 151 |
if not self.started: |
| 152 |
self.reader.read_start_array() |
| 153 |
self.started = True |
| 154 |
|
| 155 |
token = self.reader.skip_until_token_with_paths((JsonTokenType.START_MAP, "item"), (JsonTokenType.END_ARRAY, "")) |
| 156 |
if token == JsonTokenType.END_ARRAY: |
| 157 |
self.done = True |
| 158 |
raise StopIteration() |
| 159 |
|
| 160 |
frame_type = self.read_frame_type() |
| 161 |
parsed_frame = self.parse_frame(frame_type) |
| 162 |
is_primary_result = parsed_frame["FrameType"] == FrameType.DataTable and parsed_frame["TableKind"] == WellKnownDataSet.PrimaryResult.value |
| 163 |
if is_primary_result: |
| 164 |
self.started_primary_results = True |
| 165 |
elif self.started_primary_results: |
| 166 |
self.finished_primary_results = True |
| 167 |
|
| 168 |
return parsed_frame |
| 169 |
|
| 170 |
def parse_frame(self, frame_type: FrameType) -> Dict[str, Any]: |
| 171 |
if frame_type == FrameType.DataSetHeader: |
| 172 |
frame = self.extract_props(frame_type, ("IsProgressive", JsonTokenType.BOOLEAN), ("Version", JsonTokenType.STRING)) |
| 173 |
if frame["IsProgressive"]: |
| 174 |
raise KustoUnsupportedApiError.progressive_api_unsupported() |
| 175 |
return frame |
| 176 |
if frame_type in [FrameType.TableHeader, FrameType.TableFragment, FrameType.TableCompletion, FrameType.TableProgress]: |
| 177 |
raise KustoUnsupportedApiError.progressive_api_unsupported() |
| 178 |
if frame_type == FrameType.DataTable: |
| 179 |
props = self.extract_props( |
| 180 |
frame_type, |
| 181 |
("TableId", JsonTokenType.NUMBER), |
| 182 |
("TableKind", JsonTokenType.STRING), |
| 183 |
("TableName", JsonTokenType.STRING), |
| 184 |
("Columns", JsonTokenType.START_ARRAY), |
| 185 |
) |
| 186 |
self.reader.skip_until_property_name("Rows") |
| 187 |
props["Rows"] = self.row_iterator() |
| 188 |
if props["TableKind"] != WellKnownDataSet.PrimaryResult.value: |
| 189 |
props["Rows"] = list(props["Rows"]) |
| 190 |
return props |
| 191 |
if frame_type == FrameType.DataSetCompletion: |
| 192 |
res = self.extract_props(frame_type, ("HasErrors", JsonTokenType.BOOLEAN), ("Cancelled", JsonTokenType.BOOLEAN)) |
| 193 |
token = self.reader.skip_until_property_name_or_end_object("OneApiErrors") |
| 194 |
if token.token_type != JsonTokenType.END_MAP: |
| 195 |
res["OneApiErrors"] = self.parse_array(skip_start=False) |
| 196 |
return res |
| 197 |
|
| 198 |
def row_iterator(self) -> Iterator[list]: |
| 199 |
self.reader.read_token_of_type(JsonTokenType.START_ARRAY) |
| 200 |
while True: |
| 201 |
token = self.reader.read_token_of_type(JsonTokenType.START_ARRAY, JsonTokenType.END_ARRAY, JsonTokenType.START_MAP) |
| 202 |
if token.token_type == JsonTokenType.START_MAP: |
| 203 |
# Todo - this method of error handling may be problematic, since after raising an error the iteration stops. |
| 204 |
# This means that if there are more data or even more errors, we can't read them |
| 205 |
raise KustoMultiApiError([self.parse_object(skip_start=True)]) |
| 206 |
if token.token_type == JsonTokenType.END_ARRAY: |
| 207 |
return |
| 208 |
yield self.parse_array(skip_start=True) |
| 209 |
|
| 210 |
def parse_array(self, skip_start: bool) -> list: |
| 211 |
if not skip_start: |
| 212 |
self.reader.read_start_array() |
| 213 |
arr = [] |
| 214 |
|
| 215 |
while True: |
| 216 |
token = self.reader.read_token_of_type( |
| 217 |
JsonTokenType.NULL, |
| 218 |
JsonTokenType.BOOLEAN, |
| 219 |
JsonTokenType.NUMBER, |
| 220 |
JsonTokenType.STRING, |
| 221 |
JsonTokenType.START_MAP, |
| 222 |
JsonTokenType.START_ARRAY, |
| 223 |
JsonTokenType.END_ARRAY, |
| 224 |
) |
| 225 |
|
| 226 |
if token.token_type == JsonTokenType.END_ARRAY: |
| 227 |
return arr |
| 228 |
|
| 229 |
if token.token_type == JsonTokenType.START_MAP: |
| 230 |
arr.append(self.parse_object(skip_start=True)) |
| 231 |
elif token.token_type == JsonTokenType.START_ARRAY: |
| 232 |
arr.append(self.parse_array(skip_start=True)) |
| 233 |
else: |
| 234 |
arr.append(token.token_value) |
| 235 |
|
| 236 |
def parse_object(self, skip_start: bool) -> Dict[str, Any]: |
| 237 |
if not skip_start: |
| 238 |
self.reader.read_start_object() |
| 239 |
|
| 240 |
obj = {} |
| 241 |
while True: |
| 242 |
token_prop_name = self.reader.read_token_of_type(JsonTokenType.MAP_KEY, JsonTokenType.END_MAP) |
| 243 |
if token_prop_name.token_type == JsonTokenType.END_MAP: |
| 244 |
return obj |
| 245 |
prop_name = token_prop_name.token_value |
| 246 |
|
| 247 |
token = self.reader.read_token_of_type( |
| 248 |
JsonTokenType.NULL, JsonTokenType.BOOLEAN, JsonTokenType.NUMBER, JsonTokenType.STRING, JsonTokenType.START_MAP, JsonTokenType.START_ARRAY |
| 249 |
) |
| 250 |
|
| 251 |
if token.token_type == JsonTokenType.START_MAP: |
| 252 |
obj[prop_name] = self.parse_object(skip_start=True) |
| 253 |
elif token.token_type == JsonTokenType.START_ARRAY: |
| 254 |
obj[prop_name] = self.parse_array(skip_start=True) |
| 255 |
else: |
| 256 |
obj[prop_name] = token.token_value |
| 257 |
|
| 258 |
def extract_props(self, frame_type: FrameType, *props: Tuple[str, JsonTokenType]) -> Dict[str, Any]: |
| 259 |
result = {"FrameType": frame_type} |
| 260 |
props_dict = dict(props) |
| 261 |
while props_dict: |
| 262 |
name = self.reader.skip_until_any_property_name(*props_dict.keys()).token_value |
| 263 |
if props_dict[name] == JsonTokenType.START_ARRAY: |
| 264 |
result[name] = self.parse_array(skip_start=False) |
| 265 |
else: |
| 266 |
result[name] = self.reader.read_token_of_type(props_dict[name]).token_value |
| 267 |
props_dict.pop(name) |
| 268 |
|
| 269 |
return result |
| 270 |
|
| 271 |
def read_frame_type(self) -> FrameType: |
| 272 |
self.reader.skip_until_property_name("FrameType") |
| 273 |
return FrameType[self.reader.read_string()] |