Debian Codegraph @

azure-kusto-ingest/tests/test_storage_account_set.py

raw
1 import time
2 from pytest import approx
3
4 from azure.kusto.ingest._storage_account_set import _RankedStorageAccountSet
5
6 ACCOUNT_1: str = "ACCOUNT_1"
7 ACCOUNT_2: str = "ACCOUNT_2"
8 ACCOUNT_3: str = "ACCOUNT_3"
9 ACCOUNT_4: str = "ACCOUNT_4"
10 ACCOUNT_5: str = "ACCOUNT_5"
11
12
13 def create_storage_account_set(time_provider=time.time) -> _RankedStorageAccountSet:
14 storage_account_set = _RankedStorageAccountSet(time_provider=time_provider)
15 storage_account_set.add_storage_account(ACCOUNT_1)
16 storage_account_set.add_storage_account(ACCOUNT_2)
17 storage_account_set.add_storage_account(ACCOUNT_3)
18 storage_account_set.add_storage_account(ACCOUNT_4)
19 storage_account_set.add_storage_account(ACCOUNT_5)
20 return storage_account_set
21
22
23 def test_check_rank_when_no_data():
24 # When there's no success/failure data, all accounts should have the same rank
25 storage_account_set = create_storage_account_set()
26 assert storage_account_set.get_storage_account(ACCOUNT_1).get_rank() == 1
27 assert storage_account_set.get_storage_account(ACCOUNT_2).get_rank() == 1
28 assert storage_account_set.get_storage_account(ACCOUNT_3).get_rank() == 1
29 assert storage_account_set.get_storage_account(ACCOUNT_4).get_rank() == 1
30 assert storage_account_set.get_storage_account(ACCOUNT_5).get_rank() == 1
31 # Same test but with shuffled accounts
32 ranked_accounts = storage_account_set.get_ranked_shuffled_accounts()
33 for account in ranked_accounts:
34 assert account.get_rank() == 1
35 pass
36
37
38 def test_accounts_are_shuffled():
39 # Create long list of accounts and verify that they are shuffled
40 storage_account_set = _RankedStorageAccountSet()
41 for i in range(100):
42 storage_account_set.add_storage_account(f"ACCOUNT_{i}")
43 ranked_accounts_1 = storage_account_set.get_ranked_shuffled_accounts()
44 ranked_accounts_2 = storage_account_set.get_ranked_shuffled_accounts()
45 # Use set to verify same accounts are in both lists
46 a = set(ranked_accounts_1)
47 b = set(ranked_accounts_2)
48 assert a == b
49 # Use != to verify that the lists are not identical in order
50 assert ranked_accounts_1 != ranked_accounts_2
51
52
53 def test_check_rank_when_all_failure():
54 current_time = 0
55
56 def time_provider():
57 return current_time
58
59 storage_account_set = create_storage_account_set(time_provider)
60
61 # Simulate data in 10 seconds passing
62 for current_time in range(10):
63 storage_account_set.add_account_result(ACCOUNT_1, False)
64 storage_account_set.add_account_result(ACCOUNT_2, False)
65 storage_account_set.add_account_result(ACCOUNT_3, False)
66 storage_account_set.add_account_result(ACCOUNT_4, False)
67 storage_account_set.add_account_result(ACCOUNT_5, False)
68
69 # All accounts should have the same rank (0)
70 assert storage_account_set.get_storage_account(ACCOUNT_1).get_rank() == 0
71 assert storage_account_set.get_storage_account(ACCOUNT_2).get_rank() == 0
72 assert storage_account_set.get_storage_account(ACCOUNT_3).get_rank() == 0
73 assert storage_account_set.get_storage_account(ACCOUNT_4).get_rank() == 0
74 assert storage_account_set.get_storage_account(ACCOUNT_5).get_rank() == 0
75
76
77 def test_check_rank_when_success_rate_is_different():
78 current_time = 0
79
80 def time_provider():
81 return current_time
82
83 storage_account_set = create_storage_account_set(time_provider)
84
85 # Simulate data in 30 seconds passing
86 for current_time in range(60):
87 storage_account_set.add_account_result(ACCOUNT_1, True) # 100% success
88 storage_account_set.add_account_result(ACCOUNT_2, current_time % 10 != 0) # ~90% success
89 storage_account_set.add_account_result(ACCOUNT_3, current_time % 2 == 0) # ~50% success
90 storage_account_set.add_account_result(ACCOUNT_4, current_time % 3 == 0) # ~33% success
91 storage_account_set.add_account_result(ACCOUNT_5, False) # 0% success
92
93 # Get shuffled accounts
94 ranked_accounts = storage_account_set.get_ranked_shuffled_accounts()
95 # Verify that the accounts are ranked in the correct order
96 assert ranked_accounts[0].get_account_name() == ACCOUNT_1
97 assert ranked_accounts[1].get_account_name() == ACCOUNT_2
98 assert ranked_accounts[2].get_account_name() in [ACCOUNT_3, ACCOUNT_4]
99 assert ranked_accounts[3].get_account_name() in [ACCOUNT_3, ACCOUNT_4]
100 assert ranked_accounts[4].get_account_name() == ACCOUNT_5
101
102 # Verify the rank itself
103 assert ranked_accounts[0].get_rank() == 1
104 assert ranked_accounts[1].get_rank() == approx(0.9, 0.01)
105 assert storage_account_set.accounts[ACCOUNT_3].get_rank() == 0.5
106 assert storage_account_set.accounts[ACCOUNT_4].get_rank() == approx(0.32, 0.1)
107 assert ranked_accounts[4].get_rank() == 0
108
109
110 def test_old_results_count_for_less():
111 current_time = 0
112
113 def time_provider():
114 return current_time
115
116 storage_account_set = create_storage_account_set(time_provider)
117
118 storage_account_set.add_account_result(ACCOUNT_1, True)
119 current_time += 11
120 storage_account_set.add_account_result(ACCOUNT_1, True)
121 current_time += 11
122 storage_account_set.add_account_result(ACCOUNT_1, True)
123 current_time += 11
124 storage_account_set.add_account_result(ACCOUNT_1, False)
125 current_time += 11
126 storage_account_set.add_account_result(ACCOUNT_1, False)
127 current_time += 11
128 storage_account_set.add_account_result(ACCOUNT_1, False)
129
130 # rank should be smaller than 0.5 as new samples are more important
131 assert storage_account_set.accounts[ACCOUNT_1].get_rank() < 0.5
132
133
134 def test_multiple_results():
135 current_time = 0
136
137 def time_provider():
138 return current_time
139
140 storage_account_set = create_storage_account_set(time_provider)
141
142 # all results go to the same bucket
143 storage_account_set.add_account_result(ACCOUNT_1, True)
144 storage_account_set.add_account_result(ACCOUNT_1, True)
145 storage_account_set.add_account_result(ACCOUNT_1, True)
146 storage_account_set.add_account_result(ACCOUNT_1, False)
147 storage_account_set.add_account_result(ACCOUNT_1, False)
148 storage_account_set.add_account_result(ACCOUNT_1, False)
149
150 assert storage_account_set.accounts[ACCOUNT_1].get_rank() == 0.5
151
152
153 def test_buckets_are_recycled():
154 current_time = 0
155
156 def time_provider():
157 return current_time
158
159 storage_account_set = create_storage_account_set(time_provider)
160
161 storage_account_set.add_account_result(ACCOUNT_1, True)
162 current_time += 10
163 storage_account_set.add_account_result(ACCOUNT_1, True)
164 current_time += 10
165 storage_account_set.add_account_result(ACCOUNT_1, True)
166
167 # rank should be greater than 0 as we have 3 successful results
168 assert storage_account_set.accounts[ACCOUNT_1].get_rank() > 0
169
170 current_time += 10
171 storage_account_set.add_account_result(ACCOUNT_1, False)
172 current_time += 10
173 storage_account_set.add_account_result(ACCOUNT_1, False)
174 current_time += 10
175 storage_account_set.add_account_result(ACCOUNT_1, False)
176 current_time += 10
177 storage_account_set.add_account_result(ACCOUNT_1, False)
178 current_time += 10
179 storage_account_set.add_account_result(ACCOUNT_1, False)
180 current_time += 10
181 storage_account_set.add_account_result(ACCOUNT_1, False)
182
183 assert storage_account_set.accounts[ACCOUNT_1].get_rank() == 0
184
185
186 def test_big_break():
187 current_time = 0
188
189 def time_provider():
190 return current_time
191
192 storage_account_set = create_storage_account_set(time_provider)
193
194 storage_account_set.add_account_result(ACCOUNT_1, True)
195 current_time += 11
196 storage_account_set.add_account_result(ACCOUNT_1, True)
197 current_time += 11
198 storage_account_set.add_account_result(ACCOUNT_1, True)
199
200 assert storage_account_set.accounts[ACCOUNT_1].get_rank() > 0
201
202 current_time += 90
203 storage_account_set.add_account_result(ACCOUNT_1, False)
204
205 assert storage_account_set.accounts[ACCOUNT_1].get_rank() == 0