Debian Codegraph @

azure-kusto-data/azure/kusto/data/_models.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License.
3
4 import json
5 from abc import ABCMeta, abstractmethod
6 from decimal import Decimal
7 from enum import Enum
8 from typing import Iterator, List, Any, Union, Optional, Dict
9
10 from . import _converters
11 from .exceptions import KustoMultiApiError, KustoStreamingQueryError
12
13
14 class WellKnownDataSet(str, Enum):
15 """Categorizes data tables according to the role they play in the data set that a Kusto query returns."""
16
17 PrimaryResult = "PrimaryResult"
18 QueryCompletionInformation = "QueryCompletionInformation"
19 TableOfContents = "TableOfContents"
20 QueryProperties = "QueryProperties"
21
22
23 class KustoResultRow:
24 """Iterator over a Kusto result row."""
25
26 conversion_funcs = {"datetime": _converters.to_datetime, "timespan": _converters.to_timedelta, "decimal": Decimal}
27
28 def __init__(self, columns: "List[KustoResultColumn]", row: list):
29 self._value_by_name = {}
30 self._value_by_index = []
31
32 for i, value in enumerate(row):
33 column = columns[i]
34 try:
35 column_type = column.column_type.lower()
36 except AttributeError:
37 self._value_by_index.append(value)
38 self._value_by_name[columns[i]] = value
39 continue
40
41 # If you are here to read this, you probably hit some datetime/timedelta inconsistencies.
42 # Azure-Data-Explorer(Kusto) supports 7 decimal digits, while the corresponding python types supports only 6.
43 # One example why one might want this precision, is when working with pandas.
44 # In that case, use azure.kusto.data.helpers.dataframe_from_result_table which takes into account the original value.
45 typed_value = self.get_typed_value(column_type, value)
46
47 self._value_by_index.append(typed_value)
48 self._value_by_name[column.column_name] = typed_value
49
50 @staticmethod
51 def get_typed_value(column_type: str, value: Any) -> Any:
52 return KustoResultRow.conversion_funcs[column_type](value) if value is not None and column_type in KustoResultRow.conversion_funcs else value
53
54 @property
55 def columns_count(self) -> int:
56 return len(self._value_by_name)
57
58 def __iter__(self) -> Iterator[Any]:
59 for i in range(self.columns_count):
60 yield self[i]
61
62 def __getitem__(self, key: Union[str, int]) -> Any:
63 if isinstance(key, int):
64 return self._value_by_index[key]
65 return self._value_by_name[key]
66
67 def __len__(self) -> int:
68 return self.columns_count
69
70 def to_dict(self) -> Dict[str, Any]:
71 return self._value_by_name
72
73 def to_list(self) -> list:
74 return self._value_by_index
75
76 def __str__(self) -> str:
77 return "['{}']".format("', '".join([str(val) for val in self._value_by_index]))
78
79 def __repr__(self) -> str:
80 values = [repr(val) for val in self._value_by_name.values()]
81 return "KustoResultRow(['{}'], [{}])".format("', '".join(self._value_by_name), ", ".join(values))
82
83 def __eq__(self, other) -> bool:
84 if len(self) != len(other):
85 return False
86 for value_index, value in enumerate(self):
87 if value != other[value_index]:
88 return False
89 return True
90
91
92 class KustoResultColumn:
93 def __init__(self, json_column: Dict[str, Any], ordinal: int):
94 self.column_name = json_column["ColumnName"]
95 self.column_type = json_column.get("ColumnType") or json_column["DataType"]
96 self.ordinal = ordinal
97
98 def __repr__(self) -> str:
99 return "KustoResultColumn({},{})".format(json.dumps({"ColumnName": self.column_name, "ColumnType": self.column_type}), self.ordinal)
100
101
102 class BaseKustoResultTable(metaclass=ABCMeta):
103 def __init__(self, json_table: Dict[str, Any]):
104 self.table_name = json_table.get("TableName")
105 self.table_id = json_table.get("TableId")
106 self.table_kind = WellKnownDataSet[json_table["TableKind"]] if "TableKind" in json_table else None
107 self.columns = [KustoResultColumn(column, index) for index, column in enumerate(json_table["Columns"])]
108
109 self.raw_columns = json_table["Columns"]
110 self.raw_rows = json_table["Rows"]
111 self.kusto_result_rows = None
112
113 def __bool__(self) -> bool:
114 return any(self.columns)
115
116 __nonzero__ = __bool__
117
118 @property
119 def columns_count(self) -> int:
120 return len(self.columns)
121
122 @abstractmethod
123 def __len__(self) -> Optional[int]:
124 pass
125
126 @property
127 @abstractmethod
128 def rows_count(self) -> int:
129 pass
130
131
132 class BaseStreamingKustoResultTable(BaseKustoResultTable):
133 def __init__(self, json_table: Dict[str, Any]):
134 super().__init__(json_table)
135
136 self.finished = False
137 self.row_count = 0
138
139 @property
140 def rows_count(self) -> int:
141 if not self.finished:
142 raise KustoStreamingQueryError("Can't retrieve rows count before the iteration is finished")
143 return self.row_count
144
145 def __len__(self) -> Optional[int]:
146 if not self.finished:
147 return None # We return None here instead of an exception, because otherwise calling list() on the object will throw
148 return self.rows_count
149
150 def iter_rows(self) -> "BaseStreamingKustoResultTable":
151 return self
152
153
154 class KustoResultTable(BaseKustoResultTable):
155 """Iterator over a Kusto result table."""
156
157 def __init__(self, json_table: Dict[str, Any]):
158 super().__init__(json_table)
159 errors = [row for row in json_table["Rows"] if isinstance(row, dict)]
160 if errors:
161 raise KustoMultiApiError(errors)
162
163 @property
164 def rows(self) -> List[KustoResultRow]:
165 if not self.kusto_result_rows:
166 self.kusto_result_rows = [KustoResultRow(self.columns, row) for row in self.raw_rows]
167 return self.kusto_result_rows
168
169 def to_dict(self) -> Dict[str, Any]:
170 """Converts the table to a dict."""
171 return {"name": self.table_name, "kind": self.table_kind, "data": [r.to_dict() for r in self]}
172
173 @property
174 def rows_count(self) -> int:
175 return len(self.raw_rows)
176
177 def __len__(self) -> int:
178 return self.rows_count
179
180 def __iter__(self) -> Iterator[KustoResultRow]:
181 for row_index, row in enumerate(self.raw_rows):
182 if self.kusto_result_rows:
183 yield self.kusto_result_rows[row_index]
184 else:
185 yield KustoResultRow(self.columns, row)
186
187 def __getitem__(self, key: int) -> KustoResultRow:
188 return self.rows[key]
189
190 def __str__(self) -> str:
191 d = self.to_dict()
192 # enum is not serializable, using value instead
193 d["kind"] = d["kind"].value
194 return json.dumps(d, default=str)
195
196
197 class KustoStreamingResultTable(BaseStreamingKustoResultTable):
198 """
199 Iterator over a Kusto result table in streaming.
200 This class can be iterated only once.
201 """
202
203 def __next__(self) -> KustoResultRow:
204 try:
205 row = next(self.raw_rows)
206 except StopIteration:
207 self.finished = True
208 raise
209 self.row_count += 1
210 return KustoResultRow(self.columns, row)
211
212 def __iter__(self) -> Iterator[KustoResultRow]:
213 return self