Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/_status_q.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import random
4
5 from typing import List, Callable, TYPE_CHECKING
6
7 from azure.kusto.ingest._resource_manager import _ResourceUri
8 from azure.storage.queue import QueueServiceClient, QueueClient, QueueMessage, TextBase64EncodePolicy, TextBase64DecodePolicy
9
10 if TYPE_CHECKING:
11 from azure.kusto.ingest.status import StatusMessage
12
13
14 class QueueDetails:
15 def __init__(self, name, service):
16 self.name = name
17 self.service = service
18
19 def __str__(self):
20 return "QueueDetails({0.name})".format(self)
21
22
23 class StatusQueue:
24 """StatusQueue is a class to simplify access to Kusto status queues (backed by azure storage queues)."""
25
26 def __init__(self, get_queues_func: Callable[[], List[_ResourceUri]], message_cls):
27 self.get_queues_func = get_queues_func
28 self.message_cls = message_cls
29
30 def _get_queues(self) -> List[QueueClient]:
31 return [
32 QueueServiceClient(q.account_uri).get_queue_client(queue=q.object_name, message_decode_policy=TextBase64DecodePolicy())
33 for q in self.get_queues_func()
34 ]
35
36 def is_empty(self) -> bool:
37 """Checks if Status queue has any messages"""
38 return len(self.peek(1, raw=True)) == 0
39
40 def _deserialize_message(self, m: QueueMessage) -> "StatusMessage":
41 """Deserialize a message and return at as `message_cls`
42 :param m: original message m.
43 """
44 return self.message_cls(m.content)
45
46 # TODO: current implementation takes a union top n / len(queues), which is not ideal,
47 # because the user is not supposed to know that there can be multiple underlying queues
48 def peek(self, n=1, raw=False) -> List["StatusMessage"]:
49 """Peek status queue
50 :param int n: number of messages to return as part of peek.
51 :param bool raw: should message content be returned as is (no parsing).
52 """
53
54 def _peek_specific_q(_q: QueueClient, _n: int) -> bool:
55 has_messages = False
56 for m in _q.peek_messages(max_messages=_n):
57 if m:
58 has_messages = True
59 result.append(m if raw else self._deserialize_message(m))
60
61 # short circuit to prevent unneeded work
62 if len(result) == n:
63 return True
64 return has_messages
65
66 queues = self._get_queues()
67 random.shuffle(queues)
68
69 per_q = int(n / len(queues)) + 1
70
71 result = []
72
73 non_empty_qs = []
74
75 for q in queues:
76 if _peek_specific_q(q, per_q):
77 non_empty_qs.append(q)
78
79 if len(result) == n:
80 return result
81
82 # in-case queues aren't balanced, and we didn't get enough messages, iterate again and this time get all that we can
83 for q in non_empty_qs:
84 _peek_specific_q(q, n)
85 if len(result) == n:
86 return result
87
88 # because we ask for n / len(qs) + 1, we might get more message then requests
89 return result
90
91 # TODO: current implementation takes a union top n / len(queues), which is not ideal,
92 # because the user is not supposed to know that there can be multiple underlying queues
93 def pop(self, n: int = 1, raw: bool = False, delete: bool = True) -> List["StatusMessage"]:
94 """Pop status queue
95 :param int n: number of messages to return as part of peek.
96 :param bool raw: should message content be returned as is (no parsing).
97 :param bool delete: should message be deleted after pop. default is True as this is expected of a q.
98 """
99
100 def _pop_specific_q(_q: QueueClient, _n: int) -> bool:
101 has_messages = False
102 for m in _q.receive_messages(messages_per_page=_n):
103 if m:
104 has_messages = True
105 result.append(m if raw else self._deserialize_message(m))
106 if delete:
107 _q.delete_message(m.id, m.pop_receipt)
108
109 # short circuit to prevent unneeded work
110 if len(result) == n:
111 return True
112 return has_messages
113
114 queues = self._get_queues()
115 random.shuffle(queues)
116
117 per_q = int(n / len(queues)) + 1
118
119 result = []
120
121 non_empty_qs = []
122
123 for q in queues:
124 if _pop_specific_q(q, per_q):
125 non_empty_qs.append(q)
126
127 if len(result) == n:
128 return result
129
130 # in-case queues aren't balanced, and we didn't get enough messages, iterate again and this time get all that we can
131 for q in non_empty_qs:
132 _pop_specific_q(q, n)
133 if len(result) == n:
134 return result
135
136 # because we ask for n / len(qs) + 1, we might get more message then requests
137 return result