| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
from abc import ABCMeta, abstractmethod |
| 4 |
from typing import List, Iterator, Union, Dict, Any |
| 5 |
|
| 6 |
from ._models import KustoResultTable, WellKnownDataSet, KustoStreamingResultTable, BaseKustoResultTable |
| 7 |
from .exceptions import KustoStreamingQueryError |
| 8 |
from .streaming_response import StreamingDataSetEnumerator, FrameType |
| 9 |
|
| 10 |
|
| 11 |
class BaseKustoResponseDataSet(metaclass=ABCMeta): |
| 12 |
tables: list |
| 13 |
tables_count: int |
| 14 |
tables_names: list |
| 15 |
|
| 16 |
@property |
| 17 |
@abstractmethod |
| 18 |
def _error_column(self) -> str: |
| 19 |
raise NotImplementedError |
| 20 |
|
| 21 |
@property |
| 22 |
@abstractmethod |
| 23 |
def _crid_column(self) -> str: |
| 24 |
raise NotImplementedError |
| 25 |
|
| 26 |
@property |
| 27 |
@abstractmethod |
| 28 |
def _status_column(self) -> str: |
| 29 |
raise NotImplementedError |
| 30 |
|
| 31 |
@property |
| 32 |
def errors_count(self) -> int: |
| 33 |
"""Checks whether an exception was thrown.""" |
| 34 |
query_status_table = next((t for t in self.tables if t.table_kind == WellKnownDataSet.QueryCompletionInformation), None) |
| 35 |
if not query_status_table: |
| 36 |
return 0 |
| 37 |
min_level = 4 |
| 38 |
errors = 0 |
| 39 |
for row in query_status_table: |
| 40 |
if row[self._error_column] < 4: |
| 41 |
if row[self._error_column] < min_level: |
| 42 |
min_level = row[self._error_column] |
| 43 |
errors = 1 |
| 44 |
elif row[self._error_column] == min_level: |
| 45 |
errors += 1 |
| 46 |
|
| 47 |
return errors |
| 48 |
|
| 49 |
def get_exceptions(self) -> List[str]: |
| 50 |
"""Gets the exceptions retrieved from Kusto if exists.""" |
| 51 |
query_status_table = next((t for t in self.tables if t.table_kind == WellKnownDataSet.QueryCompletionInformation), None) |
| 52 |
if not query_status_table: |
| 53 |
return [] |
| 54 |
result = [] |
| 55 |
for row in query_status_table: |
| 56 |
if row[self._error_column] < 4: |
| 57 |
result.append( |
| 58 |
"Please provide the following data to Kusto: CRID='{0}' Description:'{1}'".format(row[self._crid_column], row[self._status_column]) |
| 59 |
) |
| 60 |
return result |
| 61 |
|
| 62 |
def __iter__(self) -> Iterator[BaseKustoResultTable]: |
| 63 |
return iter(self.tables) |
| 64 |
|
| 65 |
def __getitem__(self, key: Union[int, str]) -> KustoResultTable: |
| 66 |
if isinstance(key, int): |
| 67 |
return self.tables[key] |
| 68 |
try: |
| 69 |
return self.tables[self.tables_names.index(key)] |
| 70 |
except ValueError: |
| 71 |
raise LookupError(key) |
| 72 |
|
| 73 |
def __len__(self) -> int: |
| 74 |
return self.tables_count |
| 75 |
|
| 76 |
|
| 77 |
class KustoResponseDataSet(BaseKustoResponseDataSet, metaclass=ABCMeta): |
| 78 |
""" |
| 79 |
`KustoResponseDataSet` Represents the parsed data set carried by the response to a Kusto request. |
| 80 |
`KustoResponseDataSet` provides convenient methods to work with the returned result. |
| 81 |
The result table(s) are accessible via the @primary_results property. |
| 82 |
@primary_results returns a collection of `KustoResultTable`. |
| 83 |
It can contain more than one table when [`fork`](https://docs.microsoft.com/en-us/azure/kusto/query/forkoperator) is used. |
| 84 |
""" |
| 85 |
|
| 86 |
def __init__(self, json_response: List[Dict[str, Any]]): |
| 87 |
self.tables = [KustoResultTable(t) for t in json_response] |
| 88 |
self.tables_count = len(self.tables) |
| 89 |
self.tables_names = [t.table_name for t in self.tables] |
| 90 |
|
| 91 |
@property |
| 92 |
def primary_results(self) -> List[KustoResultTable]: |
| 93 |
"""Returns primary results. If there is more than one returns a list.""" |
| 94 |
if self.tables_count == 1: |
| 95 |
return self.tables |
| 96 |
primary = [x for x in self.tables if x.table_kind == WellKnownDataSet.PrimaryResult] |
| 97 |
|
| 98 |
return primary |
| 99 |
|
| 100 |
def __iter__(self) -> Iterator[KustoResultTable]: |
| 101 |
return iter(self.tables) |
| 102 |
|
| 103 |
|
| 104 |
class KustoResponseDataSetV1(KustoResponseDataSet): |
| 105 |
""" |
| 106 |
KustoResponseDataSetV1 is a wrapper for a V1 Kusto response. |
| 107 |
It parses V1 response into a convenient KustoResponseDataSet. |
| 108 |
To read more about V1 response structure, please check out https://docs.microsoft.com/en-us/azure/kusto/api/rest/response |
| 109 |
""" |
| 110 |
|
| 111 |
_status_column = "StatusDescription" |
| 112 |
_crid_column = "ClientActivityId" |
| 113 |
_error_column = "Severity" |
| 114 |
_tables_kinds = { |
| 115 |
"QueryResult": WellKnownDataSet.PrimaryResult, |
| 116 |
"QueryProperties": WellKnownDataSet.QueryProperties, |
| 117 |
"QueryStatus": WellKnownDataSet.QueryCompletionInformation, |
| 118 |
} |
| 119 |
|
| 120 |
def __init__(self, json_response: dict): |
| 121 |
super(KustoResponseDataSetV1, self).__init__(json_response["Tables"]) |
| 122 |
if self.tables_count <= 2: |
| 123 |
self.tables[0].table_kind = WellKnownDataSet.PrimaryResult |
| 124 |
self.tables[0].table_id = 0 |
| 125 |
|
| 126 |
if self.tables_count == 2: |
| 127 |
self.tables[1].table_kind = WellKnownDataSet.QueryProperties |
| 128 |
self.tables[1].table_id = 1 |
| 129 |
else: |
| 130 |
toc = self.tables[-1] |
| 131 |
toc.table_kind = WellKnownDataSet.TableOfContents |
| 132 |
toc.table_id = self.tables_count - 1 |
| 133 |
for i in range(self.tables_count - 1): |
| 134 |
self.tables[i].table_name = toc[i]["Name"] |
| 135 |
self.tables[i].table_id = toc[i]["Id"] |
| 136 |
self.tables[i].table_kind = self._tables_kinds[toc[i]["Kind"]] |
| 137 |
|
| 138 |
|
| 139 |
class KustoResponseDataSetV2(KustoResponseDataSet): |
| 140 |
""" |
| 141 |
KustoResponseDataSetV2 is a wrapper for a V2 Kusto response. |
| 142 |
It parses V2 response into a convenient KustoResponseDataSet. |
| 143 |
To read more about V2 response structure, please check out https://docs.microsoft.com/en-us/azure/kusto/api/rest/response2 |
| 144 |
""" |
| 145 |
|
| 146 |
_status_column = "Payload" |
| 147 |
_error_column = "Level" |
| 148 |
_crid_column = "ClientRequestId" |
| 149 |
|
| 150 |
def __init__(self, json_response: List[dict]): |
| 151 |
super(KustoResponseDataSetV2, self).__init__([t for t in json_response if t["FrameType"] == "DataTable"]) |
| 152 |
|
| 153 |
|
| 154 |
class KustoStreamingResponseDataSet(BaseKustoResponseDataSet): |
| 155 |
_status_column = "Payload" |
| 156 |
_error_column = "Level" |
| 157 |
_crid_column = "ClientRequestId" |
| 158 |
|
| 159 |
def __init__(self, streamed_data: StreamingDataSetEnumerator): |
| 160 |
self._current_table = None |
| 161 |
self._skip_incomplete_tables = False |
| 162 |
self.tables = [] |
| 163 |
self.streamed_data = streamed_data |
| 164 |
self.finished = False |
| 165 |
|
| 166 |
def iter_primary_results(self) -> "PrimaryResultsIterator": |
| 167 |
return PrimaryResultsIterator(self) |
| 168 |
|
| 169 |
def __iter__(self) -> Iterator[Union[KustoResultTable, KustoStreamingResultTable]]: |
| 170 |
return self |
| 171 |
|
| 172 |
def __next__(self) -> Union[KustoResultTable, KustoStreamingResultTable]: |
| 173 |
if self.finished: |
| 174 |
raise StopIteration |
| 175 |
|
| 176 |
if type(self._current_table) is KustoStreamingResultTable and not self._current_table.finished and not self._skip_incomplete_tables: |
| 177 |
raise KustoStreamingQueryError( |
| 178 |
"Tried retrieving a new primary_result table before the old one was finished. To override call `set_skip_incomplete_tables(True)`" |
| 179 |
) |
| 180 |
|
| 181 |
while True: |
| 182 |
try: |
| 183 |
table = next(self.streamed_data) |
| 184 |
except StopIteration: |
| 185 |
self.finished = True |
| 186 |
raise |
| 187 |
if table["FrameType"] == FrameType.DataTable: |
| 188 |
break |
| 189 |
|
| 190 |
if table["TableKind"] == WellKnownDataSet.PrimaryResult.value: |
| 191 |
self._current_table = KustoStreamingResultTable(table) |
| 192 |
else: |
| 193 |
self._current_table = KustoResultTable(table) |
| 194 |
|
| 195 |
self.tables.append(self._current_table) |
| 196 |
return self._current_table |
| 197 |
|
| 198 |
def set_skip_incomplete_tables(self, value: bool): |
| 199 |
self._skip_incomplete_tables = value |
| 200 |
|
| 201 |
@property |
| 202 |
def errors_count(self) -> int: |
| 203 |
if not self.finished: |
| 204 |
raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.") |
| 205 |
return super().errors_count |
| 206 |
|
| 207 |
def get_exceptions(self) -> List[str]: |
| 208 |
if not self.finished: |
| 209 |
raise KustoStreamingQueryError("Unable to get errors count before reading all of the tables.") |
| 210 |
return super().get_exceptions() |
| 211 |
|
| 212 |
def __getitem__(self, key) -> KustoResultTable: |
| 213 |
if isinstance(key, int): |
| 214 |
return self.tables[key] |
| 215 |
try: |
| 216 |
return next(t for t in self.tables if t.table_name == key) |
| 217 |
except StopIteration: |
| 218 |
raise LookupError(key) |
| 219 |
|
| 220 |
def __len__(self) -> int: |
| 221 |
return len(self.tables) |
| 222 |
|
| 223 |
|
| 224 |
class PrimaryResultsIterator: |
| 225 |
# This class exists because you can't raise exception from an generator and keep working |
| 226 |
def __init__(self, dataset: KustoStreamingResponseDataSet): |
| 227 |
self.dataset = dataset |
| 228 |
|
| 229 |
def __iter__(self) -> Iterator[KustoStreamingResultTable]: |
| 230 |
return self |
| 231 |
|
| 232 |
def __next__(self) -> KustoStreamingResultTable: |
| 233 |
while True: |
| 234 |
table = next(self.dataset) |
| 235 |
if type(table) is KustoStreamingResultTable: |
| 236 |
return table |