Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/_resource_manager.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 from datetime import datetime, timedelta
4 from typing import List, Dict
5 from urllib.parse import urlparse
6
7 from tenacity import retry_if_exception_type, stop_after_attempt, Retrying, wait_random_exponential
8
9 from azure.kusto.data import KustoClient
10 from azure.kusto.data._models import KustoResultTable
11 from azure.kusto.data._telemetry import MonitoredActivity, Span
12 from azure.kusto.data.exceptions import KustoThrottlingError
13 from azure.kusto.ingest._storage_account_set import _RankedStorageAccountSet
14
15 _SHOW_VERSION = ".show version"
16 _SERVICE_TYPE_COLUMN_NAME = "ServiceType"
17
18
19 class _ResourceUri:
20 def __init__(self, url: str):
21 self.url = url
22 self.parsed = urlparse(url)
23 self.storage_account_name = self.parsed.netloc.split(".", 1)[0]
24 self.object_name = self.parsed.path.lstrip("/")
25
26 @property
27 def account_uri(self) -> str:
28 return f"{self.parsed.scheme}://{self.parsed.netloc}/?{self.parsed.query}"
29
30 def __str__(self):
31 return self.url
32
33
34 class _IngestClientResources:
35 def __init__(
36 self,
37 secured_ready_for_aggregation_queues: List[_ResourceUri] = None,
38 failed_ingestions_queues: List[_ResourceUri] = None,
39 successful_ingestions_queues: List[_ResourceUri] = None,
40 containers: List[_ResourceUri] = None,
41 status_tables: List[_ResourceUri] = None,
42 ):
43 self.secured_ready_for_aggregation_queues = secured_ready_for_aggregation_queues
44 self.failed_ingestions_queues = failed_ingestions_queues
45 self.successful_ingestions_queues = successful_ingestions_queues
46 self.containers = containers
47 self.status_tables = status_tables
48
49 def is_applicable(self):
50 resources = [
51 self.secured_ready_for_aggregation_queues,
52 self.failed_ingestions_queues,
53 self.failed_ingestions_queues,
54 self.containers,
55 self.status_tables,
56 ]
57 return all(resources)
58
59
60 class _ResourceManager:
61 def __init__(self, kusto_client: KustoClient):
62 self._kusto_client = kusto_client
63 self._refresh_period = timedelta(hours=1)
64
65 self._ingest_client_resources = None
66 self._ingest_client_resources_last_update = None
67 self._ranked_storage_account_set = _RankedStorageAccountSet()
68
69 self._authorization_context = None
70 self._authorization_context_last_update = None
71
72 self.__set_throttling_settings()
73
74 def close(self):
75 self._kusto_client.close()
76
77 def __set_throttling_settings(self, num_of_attempts: int = 4, max_seconds_per_retry: float = 30):
78 self._retryer = Retrying(
79 wait=wait_random_exponential(max=max_seconds_per_retry),
80 retry=retry_if_exception_type(KustoThrottlingError),
81 stop=stop_after_attempt(num_of_attempts),
82 reraise=True,
83 )
84
85 def _refresh_ingest_client_resources(self):
86 if (
87 not self._ingest_client_resources
88 or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.utcnow()
89 or not self._ingest_client_resources.is_applicable()
90 ):
91 self._ingest_client_resources = self._get_ingest_client_resources_from_service()
92 self._ingest_client_resources_last_update = datetime.utcnow()
93 self._populate_ranked_storage_account_set()
94
95 def _get_resource_by_name(self, table: KustoResultTable, resource_name: str):
96 return [_ResourceUri(row["StorageRoot"]) for row in table if row["ResourceTypeName"] == resource_name]
97
98 def _get_ingest_client_resources_from_service(self):
99 # trace all calls to get ingestion resources
100 def invoker():
101 return MonitoredActivity.invoke(
102 lambda: self._kusto_client.execute("NetDefaultDB", ".get ingestion resources"),
103 name_of_span="_ResourceManager.get_ingestion_resources",
104 tracing_attributes=Span.create_cluster_attributes(self._kusto_client._kusto_cluster),
105 )
106
107 result = self._retryer(invoker)
108 table = result.primary_results[0]
109
110 secured_ready_for_aggregation_queues = self._get_resource_by_name(table, "SecuredReadyForAggregationQueue")
111 failed_ingestions_queues = self._get_resource_by_name(table, "FailedIngestionsQueue")
112 successful_ingestions_queues = self._get_resource_by_name(table, "SuccessfulIngestionsQueue")
113 containers = self._get_resource_by_name(table, "TempStorage")
114 status_tables = self._get_resource_by_name(table, "IngestionsStatusTable")
115
116 return _IngestClientResources(secured_ready_for_aggregation_queues, failed_ingestions_queues, successful_ingestions_queues, containers, status_tables)
117
118 def _refresh_authorization_context(self):
119 if (
120 not self._authorization_context
121 or self._authorization_context.isspace()
122 or (self._authorization_context_last_update + self._refresh_period) <= datetime.utcnow()
123 ):
124 self._authorization_context = self._get_authorization_context_from_service()
125 self._authorization_context_last_update = datetime.utcnow()
126
127 def _get_authorization_context_from_service(self):
128 # trace all calls to get identity token
129 def invoker():
130 return MonitoredActivity.invoke(
131 lambda: self._kusto_client.execute("NetDefaultDB", ".get kusto identity token"),
132 name_of_span="_ResourceManager.get_identity_token",
133 tracing_attributes=Span.create_cluster_attributes(self._kusto_client._kusto_cluster),
134 )
135
136 result = self._retryer(invoker)
137 return result.primary_results[0][0]["AuthorizationContext"]
138
139 def _populate_ranked_storage_account_set(self):
140 for resource in self._ingest_client_resources.containers:
141 self._ranked_storage_account_set.add_storage_account(resource.storage_account_name)
142 for resource in self._ingest_client_resources.secured_ready_for_aggregation_queues:
143 self._ranked_storage_account_set.add_storage_account(resource.storage_account_name)
144
145 def _group_resources_by_storage_account(self, resources: List[_ResourceUri]) -> Dict[str, List[_ResourceUri]]:
146 resources_by_storage_account = {}
147 for resource in resources:
148 if resource.storage_account_name not in resources_by_storage_account:
149 resources_by_storage_account[resource.storage_account_name] = list()
150 resources_by_storage_account[resource.storage_account_name].append(resource)
151
152 return resources_by_storage_account
153
154 def _get_shuffled_and_ranked_resources(self, resources: List[_ResourceUri]) -> List[List[_ResourceUri]]:
155 resources_by_storage_account = self._group_resources_by_storage_account(resources)
156 ranked_storage_accounts = self._ranked_storage_account_set.get_ranked_shuffled_accounts()
157
158 # sort resources by storage account rank
159 ranked_resources = list()
160 for storage_account in ranked_storage_accounts:
161 if storage_account.account_name in resources_by_storage_account.keys():
162 ranked_resources.append(resources_by_storage_account[storage_account.account_name])
163
164 return ranked_resources
165
166 def _shuffle_and_select_with_round_robin(self, resources: List[_ResourceUri]) -> List[_ResourceUri]:
167 # get list of resources sorted by storage account rank
168 rank_shuffled_resources_list = self._get_shuffled_and_ranked_resources(resources)
169
170 # select resources with non-repeating round robin and flatten the list
171 result = []
172 while True:
173 if all(not lst for lst in rank_shuffled_resources_list):
174 break
175
176 for lst in rank_shuffled_resources_list:
177 if lst:
178 result.append(lst.pop(0))
179
180 return result
181
182 def get_ingestion_queues(self) -> List[_ResourceUri]:
183 self._refresh_ingest_client_resources()
184 return self._shuffle_and_select_with_round_robin(self._ingest_client_resources.secured_ready_for_aggregation_queues)
185
186 def get_failed_ingestions_queues(self) -> List[_ResourceUri]:
187 self._refresh_ingest_client_resources()
188 return self._ingest_client_resources.failed_ingestions_queues
189
190 def get_successful_ingestions_queues(self) -> List[_ResourceUri]:
191 self._refresh_ingest_client_resources()
192 return self._ingest_client_resources.successful_ingestions_queues
193
194 def get_containers(self) -> List[_ResourceUri]:
195 self._refresh_ingest_client_resources()
196 return self._shuffle_and_select_with_round_robin(self._ingest_client_resources.containers)
197
198 def get_ingestions_status_tables(self) -> List[_ResourceUri]:
199 self._refresh_ingest_client_resources()
200 return self._ingest_client_resources.status_tables
201
202 def get_authorization_context(self):
203 self._refresh_authorization_context()
204 return self._authorization_context
205
206 def retrieve_service_type(self):
207 try:
208 command_result = self._kusto_client.execute("NetDefaultDB", _SHOW_VERSION)
209 return command_result.primary_results[0][0][_SERVICE_TYPE_COLUMN_NAME]
210 except (TypeError, KeyError):
211 return ""
212
213 def set_proxy(self, proxy_url: str):
214 self._kusto_client.set_proxy(proxy_url)
215
216 def report_resource_usage_result(self, storage_account_name: str, success_status: bool):
217 self._ranked_storage_account_set.add_account_result(storage_account_name, success_status)