Debian Codegraph @

azure-kusto-data/azure/kusto/data/response.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 from abc import ABCMeta, abstractmethod
4 from typing import List, Iterator, Union, Dict, Any
5
6 from ._models import KustoResultTable, WellKnownDataSet, KustoStreamingResultTable, BaseKustoResultTable
7 from .exceptions import KustoStreamingQueryError
8 from .streaming_response import StreamingDataSetEnumerator, FrameType
9
10
11 class BaseKustoResponseDataSet(metaclass=ABCMeta):
12 tables: list
13 tables_count: int
14 tables_names: list
15
16 @property
17 @abstractmethod
18 def _error_column(self) -> str:
19 raise NotImplementedError
20
21 @property
22 @abstractmethod
23 def _crid_column(self) -> str:
24 raise NotImplementedError
25
26 @property
27 @abstractmethod
28 def _status_column(self) -> str:
29 raise NotImplementedError
30
31 @property
32 def errors_count(self) -> int:
33 """Checks whether an exception was thrown."""
34 query_status_table = next((t for t in self.tables if t.table_kind == WellKnownDataSet.QueryCompletionInformation), None)
35 if not query_status_table:
36 return 0
37 min_level = 4
38 errors = 0
39 for row in query_status_table:
40 if row[self._error_column] < 4:
41 if row[self._error_column] < min_level:
42 min_level = row[self._error_column]
43 errors = 1
44 elif row[self._error_column] == min_level:
45 errors += 1
46
47 return errors
48
49 def get_exceptions(self) -> List[str]:
50 """Gets the exceptions retrieved from Kusto if exists."""
51 query_status_table = next((t for t in self.tables if t.table_kind == WellKnownDataSet.QueryCompletionInformation), None)
52 if not query_status_table:
53 return []
54 result = []
55 for row in query_status_table:
56 if row[self._error_column] < 4:
57 result.append(
58 "Please provide the following data to Kusto: CRID='{0}' Description:'{1}'".format(row[self._crid_column], row[self._status_column])
59 )
60 return result
61
62 def __iter__(self) -> Iterator[BaseKustoResultTable]:
63 return iter(self.tables)
64
65 def __getitem__(self, key: Union[int, str]) -> KustoResultTable:
66 if isinstance(key, int):
67 return self.tables[key]
68 try:
69 return self.tables[self.tables_names.index(key)]
70 except ValueError:
71 raise LookupError(key)
72
73 def __len__(self) -> int:
74 return self.tables_count
75
76
77 class KustoResponseDataSet(BaseKustoResponseDataSet, metaclass=ABCMeta):
78 """
79 `KustoResponseDataSet` Represents the parsed data set carried by the response to a Kusto request.
80 `KustoResponseDataSet` provides convenient methods to work with the returned result.
81 The result table(s) are accessible via the @primary_results property.
82 @primary_results returns a collection of `KustoResultTable`.
83 It can contain more than one table when [`fork`](https://docs.microsoft.com/en-us/azure/kusto/query/forkoperator) is used.
84 """
85
86 def __init__(self, json_response: List[Dict[str, Any]]):
87 self.tables = [KustoResultTable(t) for t in json_response]
88 self.tables_count = len(self.tables)
89 self.tables_names = [t.table_name for t in self.tables]
90
91 @property
92 def primary_results(self) -> List[KustoResultTable]:
93 """Returns primary results. If there is more than one returns a list."""
94 if self.tables_count == 1:
95 return self.tables
96 primary = [x for x in self.tables if x.table_kind == WellKnownDataSet.PrimaryResult]
97
98 return primary
99
100 def __iter__(self) -> Iterator[KustoResultTable]:
101 return iter(self.tables)
102
103
104 class KustoResponseDataSetV1(KustoResponseDataSet):
105 """
106 KustoResponseDataSetV1 is a wrapper for a V1 Kusto response.
107 It parses V1 response into a convenient KustoResponseDataSet.
108 To read more about V1 response structure, please check out https://docs.microsoft.com/en-us/azure/kusto/api/rest/response
109 """
110
111 _status_column = "StatusDescription"
112 _crid_column = "ClientActivityId"
113 _error_column = "Severity"
114 _tables_kinds = {
115 "QueryResult": WellKnownDataSet.PrimaryResult,
116 "QueryProperties": WellKnownDataSet.QueryProperties,
117 "QueryStatus": WellKnownDataSet.QueryCompletionInformation,
118 }
119
120 def __init__(self, json_response: dict):
121 super(KustoResponseDataSetV1, self).__init__(json_response["Tables"])
122 if self.tables_count <= 2:
123 self.tables[0].table_kind = WellKnownDataSet.PrimaryResult
124 self.tables[0].table_id = 0
125
126 if self.tables_count == 2:
127 self.tables[1].table_kind = WellKnownDataSet.QueryProperties
128 self.tables[1].table_id = 1
129 else:
130 toc = self.tables[-1]
131 toc.table_kind = WellKnownDataSet.TableOfContents
132 toc.table_id = self.tables_count - 1
133 for i in range(self.tables_count - 1):
134 self.tables[i].table_name = toc[i]["Name"]
135 self.tables[i].table_id = toc[i]["Id"]
136 self.tables[i].table_kind = self._tables_kinds[toc[i]["Kind"]]
137
138
139 class KustoResponseDataSetV2(KustoResponseDataSet):
140 """
141 KustoResponseDataSetV2 is a wrapper for a V2 Kusto response.
142 It parses V2 response into a convenient KustoResponseDataSet.
143 To read more about V2 response structure, please check out https://docs.microsoft.com/en-us/azure/kusto/api/rest/response2
144 """
145
146 _status_column = "Payload"
147 _error_column = "Level"
148 _crid_column = "ClientRequestId"
149
150 def __init__(self, json_response: List[dict]):
151 super(KustoResponseDataSetV2, self).__init__([t for t in json_response if t["FrameType"] == "DataTable"])
152
153
154 class KustoStreamingResponseDataSet(BaseKustoResponseDataSet):
155 _status_column = "Payload"
156 _error_column = "Level"
157 _crid_column = "ClientRequestId"
158
159 def __init__(self, streamed_data: StreamingDataSetEnumerator):
160 self._current_table = None
161 self._skip_incomplete_tables = False
162 self.tables = []
163 self.streamed_data = streamed_data
164 self.finished = False
165
166 def iter_primary_results(self) -> "PrimaryResultsIterator":
167 return PrimaryResultsIterator(self)
168
169 def __iter__(self) -> Iterator[Union[KustoResultTable, KustoStreamingResultTable]]:
170 return self
171
172 def __next__(self) -> Union[KustoResultTable, KustoStreamingResultTable]:
173 if self.finished:
174 raise StopIteration
175
176 if type(self._current_table) is KustoStreamingResultTable and not self._current_table.finished and not self._skip_incomplete_tables:
177 raise KustoStreamingQueryError(
178 "Tried retrieving a new primary_result table before the old one was finished. To override call `set_skip_incomplete_tables(True)`"
179 )
180
181 while True:
182 try:
183 table = next(self.streamed_data)
184 except StopIteration:
185 self.finished = True
186 raise
187 if table["FrameType"] == FrameType.DataTable:
188 break
189
190 if table["TableKind"] == WellKnownDataSet.PrimaryResult.value:
191 self._current_table = KustoStreamingResultTable(table)
192 else:
193 self._current_table = KustoResultTable(table)
194
195 self.tables.append(self._current_table)
196 return self._current_table
197
198 def set_skip_incomplete_tables(self, value: bool):
199 self._skip_incomplete_tables = value
200
201 @property
202 def errors_count(self) -> int:
203 if not self.finished:
204 raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.")
205 return super().errors_count
206
207 def get_exceptions(self) -> List[str]:
208 if not self.finished:
209 raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.")
210 return super().get_exceptions()
211
212 def __getitem__(self, key) -> KustoResultTable:
213 if isinstance(key, int):
214 return self.tables[key]
215 try:
216 return next(t for t in self.tables if t.table_name == key)
217 except StopIteration:
218 raise LookupError(key)
219
220 def __len__(self) -> int:
221 return len(self.tables)
222
223
224 class PrimaryResultsIterator:
225 # This class exists because you can't raise exception from an generator and keep working
226 def __init__(self, dataset: KustoStreamingResponseDataSet):
227 self.dataset = dataset
228
229 def __iter__(self) -> Iterator[KustoStreamingResultTable]:
230 return self
231
232 def __next__(self) -> KustoStreamingResultTable:
233 while True:
234 table = next(self.dataset)
235 if type(table) is KustoStreamingResultTable:
236 return table