| 1 |
# Copyright (c) Microsoft Corporation. |
| 2 |
# Licensed under the MIT License |
| 3 |
from datetime import datetime, timedelta |
| 4 |
from typing import List, Dict |
| 5 |
from urllib.parse import urlparse |
| 6 |
|
| 7 |
from tenacity import retry_if_exception_type, stop_after_attempt, Retrying, wait_random_exponential |
| 8 |
|
| 9 |
from azure.kusto.data import KustoClient |
| 10 |
from azure.kusto.data._models import KustoResultTable |
| 11 |
from azure.kusto.data._telemetry import MonitoredActivity, Span |
| 12 |
from azure.kusto.data.exceptions import KustoThrottlingError |
| 13 |
from azure.kusto.ingest._storage_account_set import _RankedStorageAccountSet |
| 14 |
|
| 15 |
_SHOW_VERSION = ".show version" |
| 16 |
_SERVICE_TYPE_COLUMN_NAME = "ServiceType" |
| 17 |
|
| 18 |
|
| 19 |
class _ResourceUri: |
| 20 |
def __init__(self, url: str): |
| 21 |
self.url = url |
| 22 |
self.parsed = urlparse(url) |
| 23 |
self.storage_account_name = self.parsed.netloc.split(".", 1)[0] |
| 24 |
self.object_name = self.parsed.path.lstrip("/") |
| 25 |
|
| 26 |
@property |
| 27 |
def account_uri(self) -> str: |
| 28 |
return f"{self.parsed.scheme}://{self.parsed.netloc}/?{self.parsed.query}" |
| 29 |
|
| 30 |
def __str__(self): |
| 31 |
return self.url |
| 32 |
|
| 33 |
|
| 34 |
class _IngestClientResources: |
| 35 |
def __init__( |
| 36 |
self, |
| 37 |
secured_ready_for_aggregation_queues: List[_ResourceUri] = None, |
| 38 |
failed_ingestions_queues: List[_ResourceUri] = None, |
| 39 |
successful_ingestions_queues: List[_ResourceUri] = None, |
| 40 |
containers: List[_ResourceUri] = None, |
| 41 |
status_tables: List[_ResourceUri] = None, |
| 42 |
): |
| 43 |
self.secured_ready_for_aggregation_queues = secured_ready_for_aggregation_queues |
| 44 |
self.failed_ingestions_queues = failed_ingestions_queues |
| 45 |
self.successful_ingestions_queues = successful_ingestions_queues |
| 46 |
self.containers = containers |
| 47 |
self.status_tables = status_tables |
| 48 |
|
| 49 |
def is_applicable(self): |
| 50 |
resources = [ |
| 51 |
self.secured_ready_for_aggregation_queues, |
| 52 |
self.failed_ingestions_queues, |
| 53 |
self.failed_ingestions_queues, |
| 54 |
self.containers, |
| 55 |
self.status_tables, |
| 56 |
] |
| 57 |
return all(resources) |
| 58 |
|
| 59 |
|
| 60 |
class _ResourceManager: |
| 61 |
def __init__(self, kusto_client: KustoClient): |
| 62 |
self._kusto_client = kusto_client |
| 63 |
self._refresh_period = timedelta(hours=1) |
| 64 |
|
| 65 |
self._ingest_client_resources = None |
| 66 |
self._ingest_client_resources_last_update = None |
| 67 |
self._ranked_storage_account_set = _RankedStorageAccountSet() |
| 68 |
|
| 69 |
self._authorization_context = None |
| 70 |
self._authorization_context_last_update = None |
| 71 |
|
| 72 |
self.__set_throttling_settings() |
| 73 |
|
| 74 |
def close(self): |
| 75 |
self._kusto_client.close() |
| 76 |
|
| 77 |
def __set_throttling_settings(self, num_of_attempts: int = 4, max_seconds_per_retry: float = 30): |
| 78 |
self._retryer = Retrying( |
| 79 |
wait=wait_random_exponential(max=max_seconds_per_retry), |
| 80 |
retry=retry_if_exception_type(KustoThrottlingError), |
| 81 |
stop=stop_after_attempt(num_of_attempts), |
| 82 |
reraise=True, |
| 83 |
) |
| 84 |
|
| 85 |
def _refresh_ingest_client_resources(self): |
| 86 |
if ( |
| 87 |
not self._ingest_client_resources |
| 88 |
or (self._ingest_client_resources_last_update + self._refresh_period) <= datetime.utcnow() |
| 89 |
or not self._ingest_client_resources.is_applicable() |
| 90 |
): |
| 91 |
self._ingest_client_resources = self._get_ingest_client_resources_from_service() |
| 92 |
self._ingest_client_resources_last_update = datetime.utcnow() |
| 93 |
self._populate_ranked_storage_account_set() |
| 94 |
|
| 95 |
def _get_resource_by_name(self, table: KustoResultTable, resource_name: str): |
| 96 |
return [_ResourceUri(row["StorageRoot"]) for row in table if row["ResourceTypeName"] == resource_name] |
| 97 |
|
| 98 |
def _get_ingest_client_resources_from_service(self): |
| 99 |
# trace all calls to get ingestion resources |
| 100 |
def invoker(): |
| 101 |
return MonitoredActivity.invoke( |
| 102 |
lambda: self._kusto_client.execute("NetDefaultDB", ".get ingestion resources"), |
| 103 |
name_of_span="_ResourceManager.get_ingestion_resources", |
| 104 |
tracing_attributes=Span.create_cluster_attributes(self._kusto_client._kusto_cluster), |
| 105 |
) |
| 106 |
|
| 107 |
result = self._retryer(invoker) |
| 108 |
table = result.primary_results[0] |
| 109 |
|
| 110 |
secured_ready_for_aggregation_queues = self._get_resource_by_name(table, "SecuredReadyForAggregationQueue") |
| 111 |
failed_ingestions_queues = self._get_resource_by_name(table, "FailedIngestionsQueue") |
| 112 |
successful_ingestions_queues = self._get_resource_by_name(table, "SuccessfulIngestionsQueue") |
| 113 |
containers = self._get_resource_by_name(table, "TempStorage") |
| 114 |
status_tables = self._get_resource_by_name(table, "IngestionsStatusTable") |
| 115 |
|
| 116 |
return _IngestClientResources(secured_ready_for_aggregation_queues, failed_ingestions_queues, successful_ingestions_queues, containers, status_tables) |
| 117 |
|
| 118 |
def _refresh_authorization_context(self): |
| 119 |
if ( |
| 120 |
not self._authorization_context |
| 121 |
or self._authorization_context.isspace() |
| 122 |
or (self._authorization_context_last_update + self._refresh_period) <= datetime.utcnow() |
| 123 |
): |
| 124 |
self._authorization_context = self._get_authorization_context_from_service() |
| 125 |
self._authorization_context_last_update = datetime.utcnow() |
| 126 |
|
| 127 |
def _get_authorization_context_from_service(self): |
| 128 |
# trace all calls to get identity token |
| 129 |
def invoker(): |
| 130 |
return MonitoredActivity.invoke( |
| 131 |
lambda: self._kusto_client.execute("NetDefaultDB", ".get kusto identity token"), |
| 132 |
name_of_span="_ResourceManager.get_identity_token", |
| 133 |
tracing_attributes=Span.create_cluster_attributes(self._kusto_client._kusto_cluster), |
| 134 |
) |
| 135 |
|
| 136 |
result = self._retryer(invoker) |
| 137 |
return result.primary_results[0][0]["AuthorizationContext"] |
| 138 |
|
| 139 |
def _populate_ranked_storage_account_set(self): |
| 140 |
for resource in self._ingest_client_resources.containers: |
| 141 |
self._ranked_storage_account_set.add_storage_account(resource.storage_account_name) |
| 142 |
for resource in self._ingest_client_resources.secured_ready_for_aggregation_queues: |
| 143 |
self._ranked_storage_account_set.add_storage_account(resource.storage_account_name) |
| 144 |
|
| 145 |
def _group_resources_by_storage_account(self, resources: List[_ResourceUri]) -> Dict[str, List[_ResourceUri]]: |
| 146 |
resources_by_storage_account = {} |
| 147 |
for resource in resources: |
| 148 |
if resource.storage_account_name not in resources_by_storage_account: |
| 149 |
resources_by_storage_account[resource.storage_account_name] = list() |
| 150 |
resources_by_storage_account[resource.storage_account_name].append(resource) |
| 151 |
|
| 152 |
return resources_by_storage_account |
| 153 |
|
| 154 |
def _get_shuffled_and_ranked_resources(self, resources: List[_ResourceUri]) -> List[List[_ResourceUri]]: |
| 155 |
resources_by_storage_account = self._group_resources_by_storage_account(resources) |
| 156 |
ranked_storage_accounts = self._ranked_storage_account_set.get_ranked_shuffled_accounts() |
| 157 |
|
| 158 |
# sort resources by storage account rank |
| 159 |
ranked_resources = list() |
| 160 |
for storage_account in ranked_storage_accounts: |
| 161 |
if storage_account.account_name in resources_by_storage_account.keys(): |
| 162 |
ranked_resources.append(resources_by_storage_account[storage_account.account_name]) |
| 163 |
|
| 164 |
return ranked_resources |
| 165 |
|
| 166 |
def _shuffle_and_select_with_round_robin(self, resources: List[_ResourceUri]) -> List[_ResourceUri]: |
| 167 |
# get list of resources sorted by storage account rank |
| 168 |
rank_shuffled_resources_list = self._get_shuffled_and_ranked_resources(resources) |
| 169 |
|
| 170 |
# select resources with non-repeating round robin and flatten the list |
| 171 |
result = [] |
| 172 |
while True: |
| 173 |
if all(not lst for lst in rank_shuffled_resources_list): |
| 174 |
break |
| 175 |
|
| 176 |
for lst in rank_shuffled_resources_list: |
| 177 |
if lst: |
| 178 |
result.append(lst.pop(0)) |
| 179 |
|
| 180 |
return result |
| 181 |
|
| 182 |
def get_ingestion_queues(self) -> List[_ResourceUri]: |
| 183 |
self._refresh_ingest_client_resources() |
| 184 |
return self._shuffle_and_select_with_round_robin(self._ingest_client_resources.secured_ready_for_aggregation_queues) |
| 185 |
|
| 186 |
def get_failed_ingestions_queues(self) -> List[_ResourceUri]: |
| 187 |
self._refresh_ingest_client_resources() |
| 188 |
return self._ingest_client_resources.failed_ingestions_queues |
| 189 |
|
| 190 |
def get_successful_ingestions_queues(self) -> List[_ResourceUri]: |
| 191 |
self._refresh_ingest_client_resources() |
| 192 |
return self._ingest_client_resources.successful_ingestions_queues |
| 193 |
|
| 194 |
def get_containers(self) -> List[_ResourceUri]: |
| 195 |
self._refresh_ingest_client_resources() |
| 196 |
return self._shuffle_and_select_with_round_robin(self._ingest_client_resources.containers) |
| 197 |
|
| 198 |
def get_ingestions_status_tables(self) -> List[_ResourceUri]: |
| 199 |
self._refresh_ingest_client_resources() |
| 200 |
return self._ingest_client_resources.status_tables |
| 201 |
|
| 202 |
def get_authorization_context(self): |
| 203 |
self._refresh_authorization_context() |
| 204 |
return self._authorization_context |
| 205 |
|
| 206 |
def retrieve_service_type(self): |
| 207 |
try: |
| 208 |
command_result = self._kusto_client.execute("NetDefaultDB", _SHOW_VERSION) |
| 209 |
return command_result.primary_results[0][0][_SERVICE_TYPE_COLUMN_NAME] |
| 210 |
except (TypeError, KeyError): |
| 211 |
return "" |
| 212 |
|
| 213 |
def set_proxy(self, proxy_url: str): |
| 214 |
self._kusto_client.set_proxy(proxy_url) |
| 215 |
|
| 216 |
def report_resource_usage_result(self, storage_account_name: str, success_status: bool): |
| 217 |
self._ranked_storage_account_set.add_account_result(storage_account_name, success_status) |