Debian Codegraph @

azure-kusto-data/azure/kusto/data/streaming_response.py

raw
1 from enum import Enum
2 from typing import Optional, Any, Tuple, Dict, AnyStr, IO, List, Iterator
3
4 import ijson
5 from ijson import IncompleteJSONError
6
7 from azure.kusto.data._models import WellKnownDataSet
8 from azure.kusto.data.exceptions import KustoServiceError, KustoTokenParsingError, KustoUnsupportedApiError, KustoApiError, KustoMultiApiError
9
10
11 class JsonTokenType(Enum):
12 NULL = 0
13 BOOLEAN = 1
14 NUMBER = 2
15 STRING = 3
16 MAP_KEY = 4
17 START_MAP = 5
18 END_MAP = 6
19 START_ARRAY = 7
20 END_ARRAY = 8
21
22 @staticmethod
23 def start_tokens() -> "List[JsonTokenType]":
24 return [JsonTokenType.START_MAP, JsonTokenType.START_ARRAY]
25
26 @staticmethod
27 def end_tokens() -> "List[JsonTokenType]":
28 return [JsonTokenType.END_MAP, JsonTokenType.END_ARRAY]
29
30
31 class FrameType(Enum):
32 DataSetHeader = 0
33 TableHeader = 1
34 TableFragment = 2
35 TableCompletion = 3
36 TableProgress = 4
37 DataTable = 5
38 DataSetCompletion = 6
39
40
41 class JsonToken:
42 def __init__(self, token_path: str, token_type: JsonTokenType, token_value: Optional[Any]):
43 self.token_path = token_path
44 self.token_type = token_type
45 self.token_value = token_value
46
47
48 class JsonTokenReader:
49 def __init__(self, stream: IO[AnyStr]):
50 self.json_iter = ijson.parse(stream, use_float=True)
51
52 def __iter__(self) -> "JsonTokenReader":
53 return self
54
55 def __next__(self) -> JsonToken:
56 return self.read_next_token_or_throw()
57
58 def read_next_token_or_throw(self) -> JsonToken:
59 try:
60 next_item = next(self.json_iter)
61 except IncompleteJSONError:
62 next_item = None
63 if next_item is None:
64 raise KustoTokenParsingError("Unexpected end of stream")
65 (token_path, token_type, token_value) = next_item
66
67 return JsonToken(token_path, JsonTokenType[token_type.upper()], token_value)
68
69 def read_token_of_type(self, *token_types: JsonTokenType) -> JsonToken:
70 token = self.read_next_token_or_throw()
71 if token.token_type not in token_types:
72 raise KustoTokenParsingError(f"Expected one the following types: '{','.join(t.name for t in token_types)}' , got type {token.token_type}")
73 return token
74
75 def read_start_object(self) -> JsonToken:
76 return self.read_token_of_type(JsonTokenType.START_MAP)
77
78 def read_start_array(self) -> JsonToken:
79 return self.read_token_of_type(JsonTokenType.START_ARRAY)
80
81 def read_string(self) -> str:
82 return self.read_token_of_type(JsonTokenType.STRING).token_value
83
84 def read_boolean(self) -> bool:
85 return self.read_token_of_type(JsonTokenType.BOOLEAN).token_value
86
87 def read_number(self) -> float:
88 return self.read_token_of_type(JsonTokenType.NUMBER).token_value
89
90 def skip_children(self, prev_token: JsonToken):
91 if prev_token.token_type == JsonTokenType.MAP_KEY:
92 prev_token = self.read_next_token_or_throw()
93
94 if prev_token.token_type in JsonTokenType.start_tokens():
95 for potential_end_token in self:
96 if potential_end_token.token_path == prev_token.token_path and potential_end_token.token_type in JsonTokenType.end_tokens():
97 break
98
99 def skip_until_property_name(self, name: str) -> JsonToken:
100 while True:
101 token = self.read_token_of_type(JsonTokenType.MAP_KEY)
102 if token.token_value == name:
103 return token
104
105 self.skip_children(token)
106
107 def skip_until_any_property_name(self, *names: str) -> JsonToken:
108 while True:
109 token = self.read_token_of_type(JsonTokenType.MAP_KEY)
110 if token.token_value in names:
111 return token
112
113 self.skip_children(token)
114
115 def skip_until_property_name_or_end_object(self, *names: str) -> JsonToken:
116 for token in self:
117 if token.token_type == JsonTokenType.END_MAP:
118 return token
119
120 if token.token_type == JsonTokenType.MAP_KEY:
121 if token.token_value in names:
122 return token
123
124 self.skip_children(token)
125 continue
126
127 raise Exception(f"Unexpected token {token}")
128
129 def skip_until_token_with_paths(self, *tokens: (JsonTokenType, str)) -> JsonToken:
130 for token in self:
131 if any((token.token_type == t_type and token.token_path == t_path) for (t_type, t_path) in tokens):
132 return token
133 self.skip_children(token)
134
135
136 class StreamingDataSetEnumerator:
137 def __init__(self, reader: JsonTokenReader):
138 self.reader = reader
139 self.done = False
140 self.started = False
141 self.started_primary_results = False
142 self.finished_primary_results = False
143
144 def __iter__(self) -> "StreamingDataSetEnumerator":
145 return self
146
147 def __next__(self) -> Dict[str, Any]:
148 if self.done:
149 raise StopIteration()
150
151 if not self.started:
152 self.reader.read_start_array()
153 self.started = True
154
155 token = self.reader.skip_until_token_with_paths((JsonTokenType.START_MAP, "item"), (JsonTokenType.END_ARRAY, ""))
156 if token == JsonTokenType.END_ARRAY:
157 self.done = True
158 raise StopIteration()
159
160 frame_type = self.read_frame_type()
161 parsed_frame = self.parse_frame(frame_type)
162 is_primary_result = parsed_frame["FrameType"] == FrameType.DataTable and parsed_frame["TableKind"] == WellKnownDataSet.PrimaryResult.value
163 if is_primary_result:
164 self.started_primary_results = True
165 elif self.started_primary_results:
166 self.finished_primary_results = True
167
168 return parsed_frame
169
170 def parse_frame(self, frame_type: FrameType) -> Dict[str, Any]:
171 if frame_type == FrameType.DataSetHeader:
172 frame = self.extract_props(frame_type, ("IsProgressive", JsonTokenType.BOOLEAN), ("Version", JsonTokenType.STRING))
173 if frame["IsProgressive"]:
174 raise KustoUnsupportedApiError.progressive_api_unsupported()
175 return frame
176 if frame_type in [FrameType.TableHeader, FrameType.TableFragment, FrameType.TableCompletion, FrameType.TableProgress]:
177 raise KustoUnsupportedApiError.progressive_api_unsupported()
178 if frame_type == FrameType.DataTable:
179 props = self.extract_props(
180 frame_type,
181 ("TableId", JsonTokenType.NUMBER),
182 ("TableKind", JsonTokenType.STRING),
183 ("TableName", JsonTokenType.STRING),
184 ("Columns", JsonTokenType.START_ARRAY),
185 )
186 self.reader.skip_until_property_name("Rows")
187 props["Rows"] = self.row_iterator()
188 if props["TableKind"] != WellKnownDataSet.PrimaryResult.value:
189 props["Rows"] = list(props["Rows"])
190 return props
191 if frame_type == FrameType.DataSetCompletion:
192 res = self.extract_props(frame_type, ("HasErrors", JsonTokenType.BOOLEAN), ("Cancelled", JsonTokenType.BOOLEAN))
193 token = self.reader.skip_until_property_name_or_end_object("OneApiErrors")
194 if token.token_type != JsonTokenType.END_MAP:
195 res["OneApiErrors"] = self.parse_array(skip_start=False)
196 return res
197
198 def row_iterator(self) -> Iterator[list]:
199 self.reader.read_token_of_type(JsonTokenType.START_ARRAY)
200 while True:
201 token = self.reader.read_token_of_type(JsonTokenType.START_ARRAY, JsonTokenType.END_ARRAY, JsonTokenType.START_MAP)
202 if token.token_type == JsonTokenType.START_MAP:
203 # Todo - this method of error handling may be problematic, since after raising an error the iteration stops.
204 # This means that if there are more data or even more errors, we can't read them
205 raise KustoMultiApiError([self.parse_object(skip_start=True)])
206 if token.token_type == JsonTokenType.END_ARRAY:
207 return
208 yield self.parse_array(skip_start=True)
209
210 def parse_array(self, skip_start: bool) -> list:
211 if not skip_start:
212 self.reader.read_start_array()
213 arr = []
214
215 while True:
216 token = self.reader.read_token_of_type(
217 JsonTokenType.NULL,
218 JsonTokenType.BOOLEAN,
219 JsonTokenType.NUMBER,
220 JsonTokenType.STRING,
221 JsonTokenType.START_MAP,
222 JsonTokenType.START_ARRAY,
223 JsonTokenType.END_ARRAY,
224 )
225
226 if token.token_type == JsonTokenType.END_ARRAY:
227 return arr
228
229 if token.token_type == JsonTokenType.START_MAP:
230 arr.append(self.parse_object(skip_start=True))
231 elif token.token_type == JsonTokenType.START_ARRAY:
232 arr.append(self.parse_array(skip_start=True))
233 else:
234 arr.append(token.token_value)
235
236 def parse_object(self, skip_start: bool) -> Dict[str, Any]:
237 if not skip_start:
238 self.reader.read_start_object()
239
240 obj = {}
241 while True:
242 token_prop_name = self.reader.read_token_of_type(JsonTokenType.MAP_KEY, JsonTokenType.END_MAP)
243 if token_prop_name.token_type == JsonTokenType.END_MAP:
244 return obj
245 prop_name = token_prop_name.token_value
246
247 token = self.reader.read_token_of_type(
248 JsonTokenType.NULL, JsonTokenType.BOOLEAN, JsonTokenType.NUMBER, JsonTokenType.STRING, JsonTokenType.START_MAP, JsonTokenType.START_ARRAY
249 )
250
251 if token.token_type == JsonTokenType.START_MAP:
252 obj[prop_name] = self.parse_object(skip_start=True)
253 elif token.token_type == JsonTokenType.START_ARRAY:
254 obj[prop_name] = self.parse_array(skip_start=True)
255 else:
256 obj[prop_name] = token.token_value
257
258 def extract_props(self, frame_type: FrameType, *props: Tuple[str, JsonTokenType]) -> Dict[str, Any]:
259 result = {"FrameType": frame_type}
260 props_dict = dict(props)
261 while props_dict:
262 name = self.reader.skip_until_any_property_name(*props_dict.keys()).token_value
263 if props_dict[name] == JsonTokenType.START_ARRAY:
264 result[name] = self.parse_array(skip_start=False)
265 else:
266 result[name] = self.reader.read_token_of_type(props_dict[name]).token_value
267 props_dict.pop(name)
268
269 return result
270
271 def read_frame_type(self) -> FrameType:
272 self.reader.skip_until_property_name("FrameType")
273 return FrameType[self.reader.read_string()]