Debian Codegraph @

azure-kusto-data/azure/kusto/data/aio/streaming_response.py

raw
1 from typing import Any, Tuple, Dict, Iterator
2
3 import aiohttp
4 import ijson
5 from ijson import IncompleteJSONError
6
7 from azure.kusto.data._models import WellKnownDataSet
8 from azure.kusto.data.exceptions import KustoTokenParsingError, KustoUnsupportedApiError, KustoApiError, KustoMultiApiError
9 from azure.kusto.data.streaming_response import JsonTokenType, FrameType, JsonToken
10
11
12 class JsonTokenReader:
13 def __init__(self, stream: aiohttp.StreamReader):
14 self.json_iter = ijson.parse_async(stream, use_float=True)
15
16 def __aiter__(self) -> "JsonTokenReader":
17 return self
18
19 def __anext__(self) -> JsonToken:
20 return self.read_next_token_or_throw()
21
22 async def read_next_token_or_throw(self) -> JsonToken:
23 try:
24 next_item = await self.json_iter.__anext__()
25 except IncompleteJSONError:
26 next_item = None
27 if next_item is None:
28 raise KustoTokenParsingError("Unexpected end of stream")
29 (token_path, token_type, token_value) = next_item
30
31 return JsonToken(token_path, JsonTokenType[token_type.upper()], token_value)
32
33 async def read_token_of_type(self, *token_types: JsonTokenType) -> JsonToken:
34 token = await self.read_next_token_or_throw()
35 if token.token_type not in token_types:
36 raise KustoTokenParsingError(f"Expected one the following types: '{','.join(t.name for t in token_types)}' , got type {token.token_type}")
37 return token
38
39 async def read_start_object(self) -> JsonToken:
40 return await self.read_token_of_type(JsonTokenType.START_MAP)
41
42 async def read_start_array(self) -> JsonToken:
43 return await self.read_token_of_type(JsonTokenType.START_ARRAY)
44
45 async def read_string(self) -> str:
46 return (await self.read_token_of_type(JsonTokenType.STRING)).token_value
47
48 async def read_boolean(self) -> bool:
49 return (await self.read_token_of_type(JsonTokenType.BOOLEAN)).token_value
50
51 async def read_number(self) -> float:
52 return (await self.read_token_of_type(JsonTokenType.NUMBER)).token_value
53
54 async def skip_children(self, prev_token: JsonToken):
55 if prev_token.token_type == JsonTokenType.MAP_KEY:
56 prev_token = await self.read_next_token_or_throw()
57 if prev_token.token_type in JsonTokenType.start_tokens():
58 async for potential_end_token in self:
59 if potential_end_token.token_path == prev_token.token_path and potential_end_token.token_type in JsonTokenType.end_tokens():
60 break
61
62 async def skip_until_property_name(self, name: str) -> JsonToken:
63 while True:
64 token = await self.read_token_of_type(JsonTokenType.MAP_KEY)
65 if token.token_value == name:
66 return token
67
68 await self.skip_children(token)
69
70 async def skip_until_any_property_name(self, *names: str) -> JsonToken:
71 while True:
72 token = await self.read_token_of_type(JsonTokenType.MAP_KEY)
73 if token.token_value in names:
74 return token
75
76 await self.skip_children(token)
77
78 async def skip_until_property_name_or_end_object(self, *names: str) -> JsonToken:
79 async for token in self:
80 if token.token_type == JsonTokenType.END_MAP:
81 return token
82
83 if token.token_type == JsonTokenType.MAP_KEY:
84 if token.token_value in names:
85 return token
86
87 await self.skip_children(token)
88 continue
89
90 raise Exception(f"Unexpected token {token}")
91
92 async def skip_until_token_with_paths(self, *tokens: (JsonTokenType, str)) -> JsonToken:
93 async for token in self:
94 if any((token.token_type == t_type and token.token_path == t_path) for (t_type, t_path) in tokens):
95 return token
96 await self.skip_children(token)
97
98
99 class StreamingDataSetEnumerator:
100 def __init__(self, reader: JsonTokenReader):
101 self.reader = reader
102 self.done = False
103 self.started = False
104 self.started_primary_results = False
105 self.finished_primary_results = False
106
107 def __aiter__(self) -> "StreamingDataSetEnumerator":
108 return self
109
110 async def __anext__(self) -> Dict[str, Any]:
111 if self.done:
112 raise StopIteration()
113
114 if not self.started:
115 await self.reader.read_start_array()
116 self.started = True
117
118 token = await self.reader.skip_until_token_with_paths((JsonTokenType.START_MAP, "item"), (JsonTokenType.END_ARRAY, ""))
119 if token == JsonTokenType.END_ARRAY:
120 self.done = True
121 raise StopIteration()
122
123 frame_type = await self.read_frame_type()
124 parsed_frame = await self.parse_frame(frame_type)
125 is_primary_result = parsed_frame["FrameType"] == FrameType.DataTable and parsed_frame["TableKind"] == WellKnownDataSet.PrimaryResult.value
126 if is_primary_result:
127 self.started_primary_results = True
128 elif self.started_primary_results:
129 self.finished_primary_results = True
130
131 return parsed_frame
132
133 async def parse_frame(self, frame_type: FrameType) -> Dict[str, Any]:
134 if frame_type == FrameType.DataSetHeader:
135 frame = await self.extract_props(frame_type, ("IsProgressive", JsonTokenType.BOOLEAN), ("Version", JsonTokenType.STRING))
136 if frame["IsProgressive"]:
137 raise KustoUnsupportedApiError.progressive_api_unsupported()
138 return frame
139 if frame_type in [FrameType.TableHeader, FrameType.TableFragment, FrameType.TableCompletion, FrameType.TableProgress]:
140 raise KustoUnsupportedApiError.progressive_api_unsupported()
141 if frame_type == FrameType.DataTable:
142 props = await self.extract_props(
143 frame_type,
144 ("TableId", JsonTokenType.NUMBER),
145 ("TableKind", JsonTokenType.STRING),
146 ("TableName", JsonTokenType.STRING),
147 ("Columns", JsonTokenType.START_ARRAY),
148 )
149 await self.reader.skip_until_property_name("Rows")
150 props["Rows"] = self.row_iterator()
151 if props["TableKind"] != WellKnownDataSet.PrimaryResult.value:
152 props["Rows"] = [r async for r in props["Rows"]]
153 return props
154 if frame_type == FrameType.DataSetCompletion:
155 res = await self.extract_props(frame_type, ("HasErrors", JsonTokenType.BOOLEAN), ("Cancelled", JsonTokenType.BOOLEAN))
156 token = await self.reader.skip_until_property_name_or_end_object("OneApiErrors")
157 if token.token_type != JsonTokenType.END_MAP:
158 res["OneApiErrors"] = self.parse_array(skip_start=False)
159 return res
160
161 async def row_iterator(self) -> Iterator[list]:
162 await self.reader.read_token_of_type(JsonTokenType.START_ARRAY)
163 while True:
164 token = await self.reader.read_token_of_type(JsonTokenType.START_ARRAY, JsonTokenType.END_ARRAY, JsonTokenType.START_MAP)
165 if token.token_type == JsonTokenType.START_MAP:
166 raise KustoMultiApiError([await self.parse_object(skip_start=True)])
167 if token.token_type == JsonTokenType.END_ARRAY:
168 return
169 yield await self.parse_array(skip_start=True)
170
171 async def parse_array(self, skip_start: bool) -> list:
172 if not skip_start:
173 await self.reader.read_start_array()
174 arr = []
175
176 while True:
177 token = await self.reader.read_token_of_type(
178 JsonTokenType.NULL,
179 JsonTokenType.BOOLEAN,
180 JsonTokenType.NUMBER,
181 JsonTokenType.STRING,
182 JsonTokenType.START_MAP,
183 JsonTokenType.START_ARRAY,
184 JsonTokenType.END_ARRAY,
185 )
186
187 if token.token_type == JsonTokenType.END_ARRAY:
188 return arr
189
190 if token.token_type == JsonTokenType.START_MAP:
191 arr.append(await self.parse_object(skip_start=True))
192 elif token.token_type == JsonTokenType.START_ARRAY:
193 arr.append(await self.parse_array(skip_start=True))
194 else:
195 arr.append(token.token_value)
196
197 async def parse_object(self, skip_start: bool) -> Dict[str, Any]:
198 if not skip_start:
199 await self.reader.read_start_object()
200
201 obj = {}
202 while True:
203 token_prop_name = await self.reader.read_token_of_type(JsonTokenType.MAP_KEY, JsonTokenType.END_MAP)
204 if token_prop_name.token_type == JsonTokenType.END_MAP:
205 return obj
206 prop_name = token_prop_name.token_value
207
208 token = await self.reader.read_token_of_type(
209 JsonTokenType.NULL, JsonTokenType.BOOLEAN, JsonTokenType.NUMBER, JsonTokenType.STRING, JsonTokenType.START_MAP, JsonTokenType.START_ARRAY
210 )
211
212 if token.token_type == JsonTokenType.START_MAP:
213 obj[prop_name] = await self.parse_object(skip_start=True)
214 elif token.token_type == JsonTokenType.START_ARRAY:
215 obj[prop_name] = await self.parse_array(skip_start=True)
216 else:
217 obj[prop_name] = token.token_value
218
219 async def extract_props(self, frame_type: FrameType, *props: Tuple[str, JsonTokenType]) -> Dict[str, Any]:
220 result = {"FrameType": frame_type}
221 props_dict = dict(props)
222 while props_dict:
223 name = (await self.reader.skip_until_any_property_name(*props_dict.keys())).token_value
224 if props_dict[name] == JsonTokenType.START_ARRAY:
225 result[name] = await self.parse_array(skip_start=False)
226 else:
227 result[name] = (await self.reader.read_token_of_type(props_dict[name])).token_value
228 props_dict.pop(name)
229
230 return result
231
232 async def read_frame_type(self) -> FrameType:
233 await self.reader.skip_until_property_name("FrameType")
234 return FrameType[await self.reader.read_string()]