Debian Codegraph @

tests/greenpool_test.py

raw
1 import gc
2 import random
3
4 import eventlet
5 from eventlet import hubs, pools
6 from eventlet.support import greenlets as greenlet
7 import tests
8
9
10 def passthru2(a, b):
11 eventlet.sleep(0.01)
12 return a, b
13
14
15 def raiser(exc):
16 raise exc
17
18
19 class GreenPool(tests.LimitedTestCase):
20 def test_spawn(self):
21 p = eventlet.GreenPool(4)
22 waiters = []
23 for i in range(10):
24 waiters.append(p.spawn(passthru, i))
25 results = [waiter.wait() for waiter in waiters]
26 self.assertEqual(results, list(range(10)))
27
28 def test_spawn_n(self):
29 p = eventlet.GreenPool(4)
30 results_closure = []
31
32 def do_something(a):
33 eventlet.sleep(0.01)
34 results_closure.append(a)
35
36 for i in range(10):
37 p.spawn(do_something, i)
38 p.waitall()
39 self.assertEqual(results_closure, list(range(10)))
40
41 def test_waiting(self):
42 pool = eventlet.GreenPool(1)
43 done = eventlet.Event()
44
45 def consume():
46 done.wait()
47
48 def waiter(pool):
49 gt = pool.spawn(consume)
50 gt.wait()
51
52 waiters = []
53 self.assertEqual(pool.running(), 0)
54 waiters.append(eventlet.spawn(waiter, pool))
55 eventlet.sleep(0)
56 self.assertEqual(pool.waiting(), 0)
57 waiters.append(eventlet.spawn(waiter, pool))
58 eventlet.sleep(0)
59 self.assertEqual(pool.waiting(), 1)
60 waiters.append(eventlet.spawn(waiter, pool))
61 eventlet.sleep(0)
62 self.assertEqual(pool.waiting(), 2)
63 self.assertEqual(pool.running(), 1)
64 done.send(None)
65 for w in waiters:
66 w.wait()
67 self.assertEqual(pool.waiting(), 0)
68 self.assertEqual(pool.running(), 0)
69
70 def test_multiple_coros(self):
71 evt = eventlet.Event()
72 results = []
73
74 def producer():
75 results.append('prod')
76 evt.send()
77
78 def consumer():
79 results.append('cons1')
80 evt.wait()
81 results.append('cons2')
82
83 pool = eventlet.GreenPool(2)
84 done = pool.spawn(consumer)
85 pool.spawn_n(producer)
86 done.wait()
87 self.assertEqual(['cons1', 'prod', 'cons2'], results)
88
89 def test_timer_cancel(self):
90 # this test verifies that local timers are not fired
91 # outside of the context of the spawn
92 timer_fired = []
93
94 def fire_timer():
95 timer_fired.append(True)
96
97 def some_work():
98 hubs.get_hub().schedule_call_local(0, fire_timer)
99
100 pool = eventlet.GreenPool(2)
101 worker = pool.spawn(some_work)
102 worker.wait()
103 eventlet.sleep(0)
104 eventlet.sleep(0)
105 self.assertEqual(timer_fired, [])
106
107 def test_reentrant(self):
108 pool = eventlet.GreenPool(1)
109
110 def reenter():
111 waiter = pool.spawn(lambda a: a, 'reenter')
112 self.assertEqual('reenter', waiter.wait())
113
114 outer_waiter = pool.spawn(reenter)
115 outer_waiter.wait()
116
117 evt = eventlet.Event()
118
119 def reenter_async():
120 pool.spawn_n(lambda a: a, 'reenter')
121 evt.send('done')
122
123 pool.spawn_n(reenter_async)
124 self.assertEqual('done', evt.wait())
125
126 def assert_pool_has_free(self, pool, num_free):
127 self.assertEqual(pool.free(), num_free)
128
129 def wait_long_time(e):
130 e.wait()
131
132 timer = eventlet.Timeout(1)
133 try:
134 evt = eventlet.Event()
135 for x in range(num_free):
136 pool.spawn(wait_long_time, evt)
137 # if the pool has fewer free than we expect,
138 # then we'll hit the timeout error
139 finally:
140 timer.cancel()
141
142 # if the runtime error is not raised it means the pool had
143 # some unexpected free items
144 timer = eventlet.Timeout(0, RuntimeError)
145 try:
146 self.assertRaises(RuntimeError, pool.spawn, wait_long_time, evt)
147 finally:
148 timer.cancel()
149
150 # clean up by causing all the wait_long_time functions to return
151 evt.send(None)
152 eventlet.sleep(0)
153 eventlet.sleep(0)
154
155 def test_resize(self):
156 pool = eventlet.GreenPool(2)
157 evt = eventlet.Event()
158
159 def wait_long_time(e):
160 e.wait()
161
162 pool.spawn(wait_long_time, evt)
163 pool.spawn(wait_long_time, evt)
164 self.assertEqual(pool.free(), 0)
165 self.assertEqual(pool.running(), 2)
166 self.assert_pool_has_free(pool, 0)
167
168 # verify that the pool discards excess items put into it
169 pool.resize(1)
170
171 # cause the wait_long_time functions to return, which will
172 # trigger puts to the pool
173 evt.send(None)
174 eventlet.sleep(0)
175 eventlet.sleep(0)
176
177 self.assertEqual(pool.free(), 1)
178 self.assertEqual(pool.running(), 0)
179 self.assert_pool_has_free(pool, 1)
180
181 # resize larger and assert that there are more free items
182 pool.resize(2)
183 self.assertEqual(pool.free(), 2)
184 self.assertEqual(pool.running(), 0)
185 self.assert_pool_has_free(pool, 2)
186
187 def test_pool_smash(self):
188 # The premise is that a coroutine in a Pool tries to get a token out
189 # of a token pool but times out before getting the token. We verify
190 # that neither pool is adversely affected by this situation.
191 pool = eventlet.GreenPool(1)
192 tp = pools.TokenPool(max_size=1)
193 tp.get() # empty out the pool
194
195 def do_receive(tp):
196 timer = eventlet.Timeout(0, RuntimeError())
197 try:
198 tp.get()
199 self.fail("Shouldn't have received anything from the pool")
200 except RuntimeError:
201 return 'timed out'
202 else:
203 timer.cancel()
204
205 # the spawn makes the token pool expect that coroutine, but then
206 # immediately cuts bait
207 e1 = pool.spawn(do_receive, tp)
208 self.assertEqual(e1.wait(), 'timed out')
209
210 # the pool can get some random item back
211 def send_wakeup(tp):
212 tp.put('wakeup')
213 gt = eventlet.spawn(send_wakeup, tp)
214
215 # now we ask the pool to run something else, which should not
216 # be affected by the previous send at all
217 def resume():
218 return 'resumed'
219 e2 = pool.spawn(resume)
220 self.assertEqual(e2.wait(), 'resumed')
221
222 # we should be able to get out the thing we put in there, too
223 self.assertEqual(tp.get(), 'wakeup')
224 gt.wait()
225
226 def test_spawn_n_2(self):
227 p = eventlet.GreenPool(2)
228 self.assertEqual(p.free(), 2)
229 r = []
230
231 def foo(a):
232 r.append(a)
233
234 gt = p.spawn(foo, 1)
235 self.assertEqual(p.free(), 1)
236 gt.wait()
237 self.assertEqual(r, [1])
238 eventlet.sleep(0)
239 self.assertEqual(p.free(), 2)
240
241 # Once the pool is exhausted, spawning forces a yield.
242 p.spawn_n(foo, 2)
243 self.assertEqual(1, p.free())
244 self.assertEqual(r, [1])
245
246 p.spawn_n(foo, 3)
247 self.assertEqual(0, p.free())
248 self.assertEqual(r, [1])
249
250 p.spawn_n(foo, 4)
251 self.assertEqual(set(r), {1, 2, 3})
252 eventlet.sleep(0)
253 self.assertEqual(set(r), {1, 2, 3, 4})
254
255 def test_exceptions(self):
256 p = eventlet.GreenPool(2)
257 for m in (p.spawn, p.spawn_n):
258 self.assert_pool_has_free(p, 2)
259 m(raiser, RuntimeError())
260 self.assert_pool_has_free(p, 1)
261 p.waitall()
262 self.assert_pool_has_free(p, 2)
263 m(raiser, greenlet.GreenletExit)
264 self.assert_pool_has_free(p, 1)
265 p.waitall()
266 self.assert_pool_has_free(p, 2)
267
268 def test_imap(self):
269 p = eventlet.GreenPool(4)
270 result_list = list(p.imap(passthru, range(10)))
271 self.assertEqual(result_list, list(range(10)))
272
273 def test_empty_imap(self):
274 p = eventlet.GreenPool(4)
275 result_iter = p.imap(passthru, [])
276 self.assertRaises(StopIteration, result_iter.next)
277
278 def test_imap_nonefunc(self):
279 p = eventlet.GreenPool(4)
280 result_list = list(p.imap(None, range(10)))
281 self.assertEqual(result_list, [(x,) for x in range(10)])
282
283 def test_imap_multi_args(self):
284 p = eventlet.GreenPool(4)
285 result_list = list(p.imap(passthru2, range(10), range(10, 20)))
286 self.assertEqual(result_list, list(zip(range(10), range(10, 20))))
287
288 def test_imap_raises(self):
289 # testing the case where the function raises an exception;
290 # both that the caller sees that exception, and that the iterator
291 # continues to be usable to get the rest of the items
292 p = eventlet.GreenPool(4)
293
294 def raiser(item):
295 if item == 1 or item == 7:
296 raise RuntimeError("intentional error")
297 else:
298 return item
299
300 it = p.imap(raiser, range(10))
301 results = []
302 while True:
303 try:
304 results.append(next(it))
305 except RuntimeError:
306 results.append('r')
307 except StopIteration:
308 break
309 self.assertEqual(results, [0, 'r', 2, 3, 4, 5, 6, 'r', 8, 9])
310
311 def test_starmap(self):
312 p = eventlet.GreenPool(4)
313 result_list = list(p.starmap(passthru, [(x,) for x in range(10)]))
314 self.assertEqual(result_list, list(range(10)))
315
316 def test_waitall_on_nothing(self):
317 p = eventlet.GreenPool()
318 p.waitall()
319
320 def test_recursive_waitall(self):
321 p = eventlet.GreenPool()
322 gt = p.spawn(p.waitall)
323 self.assertRaises(AssertionError, gt.wait)
324
325
326 class GreenPile(tests.LimitedTestCase):
327 def test_pile(self):
328 p = eventlet.GreenPile(4)
329 for i in range(10):
330 p.spawn(passthru, i)
331 result_list = list(p)
332 self.assertEqual(result_list, list(range(10)))
333
334 def test_pile_spawn_times_out(self):
335 p = eventlet.GreenPile(4)
336 for i in range(4):
337 p.spawn(passthru, i)
338 # now it should be full and this should time out
339 eventlet.Timeout(0)
340 self.assertRaises(eventlet.Timeout, p.spawn, passthru, "time out")
341 # verify that the spawn breakage didn't interrupt the sequence
342 # and terminates properly
343 for i in range(4, 10):
344 p.spawn(passthru, i)
345 self.assertEqual(list(p), list(range(10)))
346
347 def test_empty_pile(self):
348 p = eventlet.GreenPile(4)
349 # no spawn()s
350 # If this hangs, LimitedTestCase should time out
351 self.assertEqual(list(p), [])
352
353 def test_constructing_from_pool(self):
354 pool = eventlet.GreenPool(2)
355 pile1 = eventlet.GreenPile(pool)
356 pile2 = eventlet.GreenPile(pool)
357
358 def bunch_of_work(pile, unique):
359 for i in range(10):
360 pile.spawn(passthru, i + unique)
361
362 eventlet.spawn(bunch_of_work, pile1, 0)
363 eventlet.spawn(bunch_of_work, pile2, 100)
364 eventlet.sleep(0)
365 self.assertEqual(list(pile2), list(range(100, 110)))
366 self.assertEqual(list(pile1), list(range(10)))
367
368
369 def test_greenpool_type_check():
370 eventlet.GreenPool(0)
371 eventlet.GreenPool(1)
372 eventlet.GreenPool(1e3)
373
374 with tests.assert_raises(TypeError):
375 eventlet.GreenPool('foo')
376 with tests.assert_raises(ValueError):
377 eventlet.GreenPool(-1)
378
379
380 class StressException(Exception):
381 pass
382
383
384 r = random.Random(0)
385
386
387 def pressure(arg):
388 while r.random() < 0.5:
389 eventlet.sleep(r.random() * 0.001)
390 if r.random() < 0.8:
391 return arg
392 else:
393 raise StressException(arg)
394
395
396 def passthru(arg):
397 while r.random() < 0.5:
398 eventlet.sleep(r.random() * 0.001)
399 return arg
400
401
402 class Stress(tests.LimitedTestCase):
403 # tests will take extra-long
404 TEST_TIMEOUT = 60
405
406 def spawn_order_check(self, concurrency):
407 # checks that piles are strictly ordered
408 p = eventlet.GreenPile(concurrency)
409
410 def makework(count, unique):
411 for i in range(count):
412 token = (unique, i)
413 p.spawn(pressure, token)
414
415 iters = 1000
416 eventlet.spawn(makework, iters, 1)
417 eventlet.spawn(makework, iters, 2)
418 eventlet.spawn(makework, iters, 3)
419 p.spawn(pressure, (0, 0))
420 latest = [-1] * 4
421 received = 0
422 it = iter(p)
423 while True:
424 try:
425 i = next(it)
426 except StressException as exc:
427 i = exc.args[0]
428 except StopIteration:
429 break
430 received += 1
431 if received % 5 == 0:
432 eventlet.sleep(0.0001)
433 unique, order = i
434 assert latest[unique] < order
435 latest[unique] = order
436 for l in latest[1:]:
437 self.assertEqual(l, iters - 1)
438
439 def test_ordering_5(self):
440 self.spawn_order_check(5)
441
442 def test_ordering_50(self):
443 self.spawn_order_check(50)
444
445 def imap_memory_check(self, concurrency):
446 # checks that imap is strictly
447 # ordered and consumes a constant amount of memory
448 p = eventlet.GreenPool(concurrency)
449 count = 1000
450 it = p.imap(passthru, range(count))
451 latest = -1
452 while True:
453 try:
454 i = next(it)
455 except StopIteration:
456 break
457
458 if latest == -1:
459 gc.collect()
460 initial_obj_count = len(gc.get_objects())
461 assert i > latest
462 latest = i
463 if latest % 5 == 0:
464 eventlet.sleep(0.001)
465 if latest % 10 == 0:
466 gc.collect()
467 objs_created = len(gc.get_objects()) - initial_obj_count
468 assert objs_created < 25 * concurrency, objs_created
469 # make sure we got to the end
470 self.assertEqual(latest, count - 1)
471
472 def test_imap_50(self):
473 self.imap_memory_check(50)
474
475 def test_imap_500(self):
476 self.imap_memory_check(500)
477
478 def test_with_intpool(self):
479 class IntPool(pools.Pool):
480 def create(self):
481 self.current_integer = getattr(self, 'current_integer', 0) + 1
482 return self.current_integer
483
484 def subtest(intpool_size, pool_size, num_executes):
485 def run(int_pool):
486 token = int_pool.get()
487 eventlet.sleep(0.0001)
488 int_pool.put(token)
489 return token
490
491 int_pool = IntPool(max_size=intpool_size)
492 pool = eventlet.GreenPool(pool_size)
493 for ix in range(num_executes):
494 pool.spawn(run, int_pool)
495 pool.waitall()
496
497 subtest(4, 7, 7)
498 subtest(50, 75, 100)
499 for isize in (10, 20, 30, 40, 50):
500 for psize in (5, 25, 35, 50):
501 subtest(isize, psize, psize)