Debian Codegraph @

azure-kusto-ingest/tests/test_status_q.py

raw
1 # Copyright (c) Microsoft Corporation.
2 # Licensed under the MIT License
3 import json
4 import time
5 import unittest
6 import uuid
7 from unittest import mock
8 from uuid import uuid4
9
10 from azure.storage.queue import QueueMessage, QueueClient
11
12 from azure.kusto.ingest import QueuedIngestClient, IngestionResult, IngestionStatus
13 from azure.kusto.ingest._resource_manager import _ResourceUri
14 from azure.kusto.ingest.status import KustoIngestStatusQueues, SuccessMessage, FailureMessage
15
16 SAS = "sp=rl&st=2020-05-20T13:38:37Z&se=2020-05-21T13:38:37Z&sv=2019-10-10&sr=c&sig=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
17 ENDPOINT_SUFFIX = "core.windows.net"
18
19
20 def mock_message(success):
21 m = QueueMessage()
22 m.id = uuid4()
23 m.insertion_time = time.time()
24 m.expiration_time = None
25 m.dequeue_count = None
26
27 if success:
28 content = {
29 "OperationId": str(m.id),
30 "Database": "db1",
31 "Table": "table1",
32 "IngestionSourceId": str(m.id),
33 "IngestionSourcePath": "blob/path",
34 "RootActivityId": "1",
35 "SucceededOn": time.time(),
36 }
37 else:
38 content = {
39 "OperationId": str(m.id),
40 "Database": "db1",
41 "Table": "table1",
42 "IngestionSourceId": str(m.id),
43 "IngestionSourcePath": "blob/path",
44 "RootActivityId": "1",
45 "FailedOn": time.time(),
46 "Details": "",
47 "ErrorCode": "1",
48 "FailureStatus": "",
49 "OriginatesFromUpdatePolicy": "",
50 "ShouldRetry": False,
51 }
52
53 m.content = json.dumps(content)
54 m.pop_receipt = None
55 m.time_next_visible = None
56
57 return m
58
59
60 def fake_peek_factory(f):
61 def fake_peek(self, max_messages):
62 return f(self.queue_name, max_messages)
63
64 return fake_peek
65
66
67 def fake_receive_factory(f):
68 def fake_receive(self, messages_per_page, *args, **kwargs):
69 return f(self.queue_name, messages_per_page)
70
71 return fake_receive
72
73
74 def fake_delete_factory(f):
75 def fake_delete(self, n):
76 return f(self.queue_name, n)
77
78 return fake_delete
79
80
81 def get_resource_uri(account: str, container: str) -> _ResourceUri:
82 return _ResourceUri("https://{0}.{1}.{2}/{3}?{4}".format(account, "queue", ENDPOINT_SUFFIX, container, SAS))
83
84
85 class StatusQTests(unittest.TestCase):
86 def test_init(self):
87 client = QueuedIngestClient("some-cluster")
88 qs = KustoIngestStatusQueues(client)
89
90 assert qs.success.message_cls == SuccessMessage
91 assert qs.failure.message_cls == FailureMessage
92
93 def test_isempty(self):
94 client = QueuedIngestClient("some-cluster")
95
96 fake_peek = fake_peek_factory(
97 lambda queue_name, num_messages=1: [mock_message(success=True) for _ in range(0, num_messages)] if "qs" in queue_name else []
98 )
99 with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues") as mocked_get_success_qs, mock.patch.object(
100 client._resource_manager, "get_failed_ingestions_queues"
101 ) as mocked_get_failed_qs, mock.patch.object(QueueClient, "peek_messages", autospec=True, side_effect=fake_peek) as q_mock:
102 fake_failed_queue = get_resource_uri("mocked_storage_account_f1", "mocked_qf_name")
103 fake_success_queue = get_resource_uri("mocked_storage_account2", "mocked_qs_name")
104
105 mocked_get_success_qs.return_value = [fake_success_queue]
106 mocked_get_failed_qs.return_value = [fake_failed_queue]
107
108 qs = KustoIngestStatusQueues(client)
109
110 assert qs.success.is_empty() is False
111 assert qs.failure.is_empty() is True
112
113 assert q_mock.call_count == 2
114 assert q_mock.call_args_list[0][1]["max_messages"] == 2
115 assert q_mock.call_args_list[1][1]["max_messages"] == 2
116
117 def test_peek(self):
118 client = QueuedIngestClient("some-cluster")
119
120 fake_peek = fake_peek_factory(
121 lambda queue_name, num_messages=1: [
122 mock_message(success=True) if "qs" in queue_name else mock_message(success=False) for _ in range(0, num_messages)
123 ]
124 )
125
126 with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues") as mocked_get_success_qs, mock.patch.object(
127 client._resource_manager, "get_failed_ingestions_queues"
128 ) as mocked_get_failed_qs, mock.patch.object(QueueClient, "peek_messages", autospec=True, side_effect=fake_peek) as q_mock:
129 fake_failed_queue1 = get_resource_uri("mocked_storage_account_f1", "mocked_qf_name")
130 fake_failed_queue2 = get_resource_uri("mocked_storage_account_f2", "mocked_qf_2_name")
131 fake_success_queue = get_resource_uri("mocked_storage_account2", "mocked_qs_name")
132
133 mocked_get_success_qs.return_value = [fake_success_queue]
134 mocked_get_failed_qs.return_value = [fake_failed_queue1, fake_failed_queue2]
135
136 qs = KustoIngestStatusQueues(client)
137
138 peek_success_actual = qs.success.peek()
139 peek_failure_actual = qs.failure.peek(6)
140
141 assert len(peek_success_actual) == 1
142
143 for m in peek_failure_actual:
144 assert isinstance(m, FailureMessage) is True
145
146 for m in peek_success_actual:
147 assert isinstance(m, SuccessMessage) is True
148
149 assert len(peek_failure_actual) == 6
150
151 actual = {}
152
153 assert len(QueueClient.peek_messages.call_args_list) == 3
154
155 for call_args in q_mock.call_args_list:
156 actual[call_args[0][0].queue_name] = actual.get(call_args[0][0].queue_name, 0) + call_args[1]["max_messages"]
157
158 assert actual[fake_failed_queue2.object_name] == 4
159 assert actual[fake_failed_queue1.object_name] == 4
160 assert actual[fake_success_queue.object_name] == 2
161
162 def test_pop(self):
163 client = QueuedIngestClient("some-cluster")
164
165 fake_receive = fake_receive_factory(
166 lambda queue_name, num_messages=1: [
167 mock_message(success=True) if "qs" in queue_name else mock_message(success=False) for _ in range(0, num_messages)
168 ]
169 )
170
171 with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues") as mocked_get_success_qs, mock.patch.object(
172 client._resource_manager, "get_failed_ingestions_queues"
173 ) as mocked_get_failed_qs, mock.patch.object(
174 QueueClient,
175 "receive_messages",
176 autospec=True,
177 side_effect=fake_receive,
178 ) as q_receive_mock, mock.patch.object(
179 QueueClient, "delete_message", return_value=None
180 ) as q_del_mock:
181 fake_failed_queue1 = get_resource_uri("mocked_storage_account_f1", "mocked_qf_name")
182 fake_failed_queue2 = get_resource_uri("mocked_storage_account_f2", "mocked_qf_2_name")
183 fake_success_queue = get_resource_uri("mocked_storage_account2", "mocked_qs_name")
184
185 mocked_get_success_qs.return_value = [fake_success_queue]
186 mocked_get_failed_qs.return_value = [fake_failed_queue1, fake_failed_queue2]
187
188 qs = KustoIngestStatusQueues(client)
189
190 get_success_actual = qs.success.pop()
191 get_failure_actual = qs.failure.pop(6)
192
193 assert len(get_success_actual) == 1
194 assert len(get_failure_actual) == 6
195
196 for m in get_failure_actual:
197 assert isinstance(m, FailureMessage)
198
199 for m in get_success_actual:
200 assert isinstance(m, SuccessMessage)
201
202 assert q_receive_mock.call_count == 3
203 assert q_del_mock.call_count == len(get_success_actual) + len(get_failure_actual)
204
205 assert q_receive_mock.call_args_list[0][1]["messages_per_page"] == 2
206
207 actual = {
208 q_receive_mock.call_args_list[1][0][0].queue_name: q_receive_mock.call_args_list[1][1]["messages_per_page"],
209 q_receive_mock.call_args_list[2][0][0].queue_name: q_receive_mock.call_args_list[2][1]["messages_per_page"],
210 }
211
212 assert actual[fake_failed_queue2.object_name] == 4
213 assert actual[fake_failed_queue1.object_name] == 4
214
215 client.close()
216
217 def test_pop_unbalanced_queues(self):
218 client = QueuedIngestClient("some-cluster")
219
220 fake_receive = fake_receive_factory(
221 lambda queue_name, messages_per_page=1: [mock_message(success=False) for _ in range(0, messages_per_page)] if "1" in queue_name else []
222 )
223 with mock.patch.object(client._resource_manager, "get_successful_ingestions_queues"), mock.patch.object(
224 client._resource_manager, "get_failed_ingestions_queues"
225 ) as mocked_get_failed_qs, mock.patch.object(
226 QueueClient,
227 "receive_messages",
228 autospec=True,
229 side_effect=fake_receive,
230 ) as q_receive_mock, mock.patch.object(
231 QueueClient, "delete_message", return_value=None
232 ):
233 fake_failed_queue1 = get_resource_uri("mocked_storage_account_f1", "mocked_qf_1_name")
234 fake_failed_queue2 = get_resource_uri("mocked_storage_account_f2", "mocked_qf_2_name")
235
236 mocked_get_failed_qs.return_value = [fake_failed_queue1, fake_failed_queue2]
237
238 qs = KustoIngestStatusQueues(client)
239
240 get_failure_actual = qs.failure.pop(6)
241
242 assert len(get_failure_actual) == 6
243
244 for m in get_failure_actual:
245 assert isinstance(m, FailureMessage)
246
247 assert q_receive_mock.call_count == 3
248
249 actual = {}
250
251 for call_args in q_receive_mock.call_args_list:
252 actual[call_args[0][0].queue_name] = actual.get(call_args[0][0].queue_name, 0) + call_args[1]["messages_per_page"]
253
254 assert actual[fake_failed_queue2.object_name] + actual[fake_failed_queue1.object_name] == (4 + 4 + 6)
255
256 def test_status_sanity(self):
257 i = IngestionResult(IngestionStatus.QUEUED, "db", "table", uuid.UUID("11111111-1111-1111-1111-111111111111"), "path")
258 assert (
259 repr(i)
260 == "IngestionResult(status=IngestionStatus.QUEUED, database=db, table=table, source_id=11111111-1111-1111-1111-111111111111, obfuscated_blob_uri=path)"
261 )
262
263 no_path = IngestionResult(IngestionStatus.QUEUED, "db", "table", uuid.UUID("11111111-1111-1111-1111-111111111111"))
264 assert repr(no_path) == "IngestionResult(status=IngestionStatus.QUEUED, database=db, table=table, source_id=11111111-1111-1111-1111-111111111111)"