| 1 |
import gc |
| 2 |
import random |
| 3 |
|
| 4 |
import eventlet |
| 5 |
from eventlet import hubs, pools |
| 6 |
from eventlet.support import greenlets as greenlet |
| 7 |
import tests |
| 8 |
|
| 9 |
|
| 10 |
def passthru2(a, b): |
| 11 |
eventlet.sleep(0.01) |
| 12 |
return a, b |
| 13 |
|
| 14 |
|
| 15 |
def raiser(exc): |
| 16 |
raise exc |
| 17 |
|
| 18 |
|
| 19 |
class GreenPool(tests.LimitedTestCase): |
| 20 |
def test_spawn(self): |
| 21 |
p = eventlet.GreenPool(4) |
| 22 |
waiters = [] |
| 23 |
for i in range(10): |
| 24 |
waiters.append(p.spawn(passthru, i)) |
| 25 |
results = [waiter.wait() for waiter in waiters] |
| 26 |
self.assertEqual(results, list(range(10))) |
| 27 |
|
| 28 |
def test_spawn_n(self): |
| 29 |
p = eventlet.GreenPool(4) |
| 30 |
results_closure = [] |
| 31 |
|
| 32 |
def do_something(a): |
| 33 |
eventlet.sleep(0.01) |
| 34 |
results_closure.append(a) |
| 35 |
|
| 36 |
for i in range(10): |
| 37 |
p.spawn(do_something, i) |
| 38 |
p.waitall() |
| 39 |
self.assertEqual(results_closure, list(range(10))) |
| 40 |
|
| 41 |
def test_waiting(self): |
| 42 |
pool = eventlet.GreenPool(1) |
| 43 |
done = eventlet.Event() |
| 44 |
|
| 45 |
def consume(): |
| 46 |
done.wait() |
| 47 |
|
| 48 |
def waiter(pool): |
| 49 |
gt = pool.spawn(consume) |
| 50 |
gt.wait() |
| 51 |
|
| 52 |
waiters = [] |
| 53 |
self.assertEqual(pool.running(), 0) |
| 54 |
waiters.append(eventlet.spawn(waiter, pool)) |
| 55 |
eventlet.sleep(0) |
| 56 |
self.assertEqual(pool.waiting(), 0) |
| 57 |
waiters.append(eventlet.spawn(waiter, pool)) |
| 58 |
eventlet.sleep(0) |
| 59 |
self.assertEqual(pool.waiting(), 1) |
| 60 |
waiters.append(eventlet.spawn(waiter, pool)) |
| 61 |
eventlet.sleep(0) |
| 62 |
self.assertEqual(pool.waiting(), 2) |
| 63 |
self.assertEqual(pool.running(), 1) |
| 64 |
done.send(None) |
| 65 |
for w in waiters: |
| 66 |
w.wait() |
| 67 |
self.assertEqual(pool.waiting(), 0) |
| 68 |
self.assertEqual(pool.running(), 0) |
| 69 |
|
| 70 |
def test_multiple_coros(self): |
| 71 |
evt = eventlet.Event() |
| 72 |
results = [] |
| 73 |
|
| 74 |
def producer(): |
| 75 |
results.append('prod') |
| 76 |
evt.send() |
| 77 |
|
| 78 |
def consumer(): |
| 79 |
results.append('cons1') |
| 80 |
evt.wait() |
| 81 |
results.append('cons2') |
| 82 |
|
| 83 |
pool = eventlet.GreenPool(2) |
| 84 |
done = pool.spawn(consumer) |
| 85 |
pool.spawn_n(producer) |
| 86 |
done.wait() |
| 87 |
self.assertEqual(['cons1', 'prod', 'cons2'], results) |
| 88 |
|
| 89 |
def test_timer_cancel(self): |
| 90 |
# this test verifies that local timers are not fired |
| 91 |
# outside of the context of the spawn |
| 92 |
timer_fired = [] |
| 93 |
|
| 94 |
def fire_timer(): |
| 95 |
timer_fired.append(True) |
| 96 |
|
| 97 |
def some_work(): |
| 98 |
hubs.get_hub().schedule_call_local(0, fire_timer) |
| 99 |
|
| 100 |
pool = eventlet.GreenPool(2) |
| 101 |
worker = pool.spawn(some_work) |
| 102 |
worker.wait() |
| 103 |
eventlet.sleep(0) |
| 104 |
eventlet.sleep(0) |
| 105 |
self.assertEqual(timer_fired, []) |
| 106 |
|
| 107 |
def test_reentrant(self): |
| 108 |
pool = eventlet.GreenPool(1) |
| 109 |
|
| 110 |
def reenter(): |
| 111 |
waiter = pool.spawn(lambda a: a, 'reenter') |
| 112 |
self.assertEqual('reenter', waiter.wait()) |
| 113 |
|
| 114 |
outer_waiter = pool.spawn(reenter) |
| 115 |
outer_waiter.wait() |
| 116 |
|
| 117 |
evt = eventlet.Event() |
| 118 |
|
| 119 |
def reenter_async(): |
| 120 |
pool.spawn_n(lambda a: a, 'reenter') |
| 121 |
evt.send('done') |
| 122 |
|
| 123 |
pool.spawn_n(reenter_async) |
| 124 |
self.assertEqual('done', evt.wait()) |
| 125 |
|
| 126 |
def assert_pool_has_free(self, pool, num_free): |
| 127 |
self.assertEqual(pool.free(), num_free) |
| 128 |
|
| 129 |
def wait_long_time(e): |
| 130 |
e.wait() |
| 131 |
|
| 132 |
timer = eventlet.Timeout(1) |
| 133 |
try: |
| 134 |
evt = eventlet.Event() |
| 135 |
for x in range(num_free): |
| 136 |
pool.spawn(wait_long_time, evt) |
| 137 |
# if the pool has fewer free than we expect, |
| 138 |
# then we'll hit the timeout error |
| 139 |
finally: |
| 140 |
timer.cancel() |
| 141 |
|
| 142 |
# if the runtime error is not raised it means the pool had |
| 143 |
# some unexpected free items |
| 144 |
timer = eventlet.Timeout(0, RuntimeError) |
| 145 |
try: |
| 146 |
self.assertRaises(RuntimeError, pool.spawn, wait_long_time, evt) |
| 147 |
finally: |
| 148 |
timer.cancel() |
| 149 |
|
| 150 |
# clean up by causing all the wait_long_time functions to return |
| 151 |
evt.send(None) |
| 152 |
eventlet.sleep(0) |
| 153 |
eventlet.sleep(0) |
| 154 |
|
| 155 |
def test_resize(self): |
| 156 |
pool = eventlet.GreenPool(2) |
| 157 |
evt = eventlet.Event() |
| 158 |
|
| 159 |
def wait_long_time(e): |
| 160 |
e.wait() |
| 161 |
|
| 162 |
pool.spawn(wait_long_time, evt) |
| 163 |
pool.spawn(wait_long_time, evt) |
| 164 |
self.assertEqual(pool.free(), 0) |
| 165 |
self.assertEqual(pool.running(), 2) |
| 166 |
self.assert_pool_has_free(pool, 0) |
| 167 |
|
| 168 |
# verify that the pool discards excess items put into it |
| 169 |
pool.resize(1) |
| 170 |
|
| 171 |
# cause the wait_long_time functions to return, which will |
| 172 |
# trigger puts to the pool |
| 173 |
evt.send(None) |
| 174 |
eventlet.sleep(0) |
| 175 |
eventlet.sleep(0) |
| 176 |
|
| 177 |
self.assertEqual(pool.free(), 1) |
| 178 |
self.assertEqual(pool.running(), 0) |
| 179 |
self.assert_pool_has_free(pool, 1) |
| 180 |
|
| 181 |
# resize larger and assert that there are more free items |
| 182 |
pool.resize(2) |
| 183 |
self.assertEqual(pool.free(), 2) |
| 184 |
self.assertEqual(pool.running(), 0) |
| 185 |
self.assert_pool_has_free(pool, 2) |
| 186 |
|
| 187 |
def test_pool_smash(self): |
| 188 |
# The premise is that a coroutine in a Pool tries to get a token out |
| 189 |
# of a token pool but times out before getting the token. We verify |
| 190 |
# that neither pool is adversely affected by this situation. |
| 191 |
pool = eventlet.GreenPool(1) |
| 192 |
tp = pools.TokenPool(max_size=1) |
| 193 |
tp.get() # empty out the pool |
| 194 |
|
| 195 |
def do_receive(tp): |
| 196 |
timer = eventlet.Timeout(0, RuntimeError()) |
| 197 |
try: |
| 198 |
tp.get() |
| 199 |
self.fail("Shouldn't have received anything from the pool") |
| 200 |
except RuntimeError: |
| 201 |
return 'timed out' |
| 202 |
else: |
| 203 |
timer.cancel() |
| 204 |
|
| 205 |
# the spawn makes the token pool expect that coroutine, but then |
| 206 |
# immediately cuts bait |
| 207 |
e1 = pool.spawn(do_receive, tp) |
| 208 |
self.assertEqual(e1.wait(), 'timed out') |
| 209 |
|
| 210 |
# the pool can get some random item back |
| 211 |
def send_wakeup(tp): |
| 212 |
tp.put('wakeup') |
| 213 |
gt = eventlet.spawn(send_wakeup, tp) |
| 214 |
|
| 215 |
# now we ask the pool to run something else, which should not |
| 216 |
# be affected by the previous send at all |
| 217 |
def resume(): |
| 218 |
return 'resumed' |
| 219 |
e2 = pool.spawn(resume) |
| 220 |
self.assertEqual(e2.wait(), 'resumed') |
| 221 |
|
| 222 |
# we should be able to get out the thing we put in there, too |
| 223 |
self.assertEqual(tp.get(), 'wakeup') |
| 224 |
gt.wait() |
| 225 |
|
| 226 |
def test_spawn_n_2(self): |
| 227 |
p = eventlet.GreenPool(2) |
| 228 |
self.assertEqual(p.free(), 2) |
| 229 |
r = [] |
| 230 |
|
| 231 |
def foo(a): |
| 232 |
r.append(a) |
| 233 |
|
| 234 |
gt = p.spawn(foo, 1) |
| 235 |
self.assertEqual(p.free(), 1) |
| 236 |
gt.wait() |
| 237 |
self.assertEqual(r, [1]) |
| 238 |
eventlet.sleep(0) |
| 239 |
self.assertEqual(p.free(), 2) |
| 240 |
|
| 241 |
# Once the pool is exhausted, spawning forces a yield. |
| 242 |
p.spawn_n(foo, 2) |
| 243 |
self.assertEqual(1, p.free()) |
| 244 |
self.assertEqual(r, [1]) |
| 245 |
|
| 246 |
p.spawn_n(foo, 3) |
| 247 |
self.assertEqual(0, p.free()) |
| 248 |
self.assertEqual(r, [1]) |
| 249 |
|
| 250 |
p.spawn_n(foo, 4) |
| 251 |
self.assertEqual(set(r), {1, 2, 3}) |
| 252 |
eventlet.sleep(0) |
| 253 |
self.assertEqual(set(r), {1, 2, 3, 4}) |
| 254 |
|
| 255 |
def test_exceptions(self): |
| 256 |
p = eventlet.GreenPool(2) |
| 257 |
for m in (p.spawn, p.spawn_n): |
| 258 |
self.assert_pool_has_free(p, 2) |
| 259 |
m(raiser, RuntimeError()) |
| 260 |
self.assert_pool_has_free(p, 1) |
| 261 |
p.waitall() |
| 262 |
self.assert_pool_has_free(p, 2) |
| 263 |
m(raiser, greenlet.GreenletExit) |
| 264 |
self.assert_pool_has_free(p, 1) |
| 265 |
p.waitall() |
| 266 |
self.assert_pool_has_free(p, 2) |
| 267 |
|
| 268 |
def test_imap(self): |
| 269 |
p = eventlet.GreenPool(4) |
| 270 |
result_list = list(p.imap(passthru, range(10))) |
| 271 |
self.assertEqual(result_list, list(range(10))) |
| 272 |
|
| 273 |
def test_empty_imap(self): |
| 274 |
p = eventlet.GreenPool(4) |
| 275 |
result_iter = p.imap(passthru, []) |
| 276 |
self.assertRaises(StopIteration, result_iter.next) |
| 277 |
|
| 278 |
def test_imap_nonefunc(self): |
| 279 |
p = eventlet.GreenPool(4) |
| 280 |
result_list = list(p.imap(None, range(10))) |
| 281 |
self.assertEqual(result_list, [(x,) for x in range(10)]) |
| 282 |
|
| 283 |
def test_imap_multi_args(self): |
| 284 |
p = eventlet.GreenPool(4) |
| 285 |
result_list = list(p.imap(passthru2, range(10), range(10, 20))) |
| 286 |
self.assertEqual(result_list, list(zip(range(10), range(10, 20)))) |
| 287 |
|
| 288 |
def test_imap_raises(self): |
| 289 |
# testing the case where the function raises an exception; |
| 290 |
# both that the caller sees that exception, and that the iterator |
| 291 |
# continues to be usable to get the rest of the items |
| 292 |
p = eventlet.GreenPool(4) |
| 293 |
|
| 294 |
def raiser(item): |
| 295 |
if item == 1 or item == 7: |
| 296 |
raise RuntimeError("intentional error") |
| 297 |
else: |
| 298 |
return item |
| 299 |
|
| 300 |
it = p.imap(raiser, range(10)) |
| 301 |
results = [] |
| 302 |
while True: |
| 303 |
try: |
| 304 |
results.append(next(it)) |
| 305 |
except RuntimeError: |
| 306 |
results.append('r') |
| 307 |
except StopIteration: |
| 308 |
break |
| 309 |
self.assertEqual(results, [0, 'r', 2, 3, 4, 5, 6, 'r', 8, 9]) |
| 310 |
|
| 311 |
def test_starmap(self): |
| 312 |
p = eventlet.GreenPool(4) |
| 313 |
result_list = list(p.starmap(passthru, [(x,) for x in range(10)])) |
| 314 |
self.assertEqual(result_list, list(range(10))) |
| 315 |
|
| 316 |
def test_waitall_on_nothing(self): |
| 317 |
p = eventlet.GreenPool() |
| 318 |
p.waitall() |
| 319 |
|
| 320 |
def test_recursive_waitall(self): |
| 321 |
p = eventlet.GreenPool() |
| 322 |
gt = p.spawn(p.waitall) |
| 323 |
self.assertRaises(AssertionError, gt.wait) |
| 324 |
|
| 325 |
|
| 326 |
class GreenPile(tests.LimitedTestCase): |
| 327 |
def test_pile(self): |
| 328 |
p = eventlet.GreenPile(4) |
| 329 |
for i in range(10): |
| 330 |
p.spawn(passthru, i) |
| 331 |
result_list = list(p) |
| 332 |
self.assertEqual(result_list, list(range(10))) |
| 333 |
|
| 334 |
def test_pile_spawn_times_out(self): |
| 335 |
p = eventlet.GreenPile(4) |
| 336 |
for i in range(4): |
| 337 |
p.spawn(passthru, i) |
| 338 |
# now it should be full and this should time out |
| 339 |
eventlet.Timeout(0) |
| 340 |
self.assertRaises(eventlet.Timeout, p.spawn, passthru, "time out") |
| 341 |
# verify that the spawn breakage didn't interrupt the sequence |
| 342 |
# and terminates properly |
| 343 |
for i in range(4, 10): |
| 344 |
p.spawn(passthru, i) |
| 345 |
self.assertEqual(list(p), list(range(10))) |
| 346 |
|
| 347 |
def test_empty_pile(self): |
| 348 |
p = eventlet.GreenPile(4) |
| 349 |
# no spawn()s |
| 350 |
# If this hangs, LimitedTestCase should time out |
| 351 |
self.assertEqual(list(p), []) |
| 352 |
|
| 353 |
def test_constructing_from_pool(self): |
| 354 |
pool = eventlet.GreenPool(2) |
| 355 |
pile1 = eventlet.GreenPile(pool) |
| 356 |
pile2 = eventlet.GreenPile(pool) |
| 357 |
|
| 358 |
def bunch_of_work(pile, unique): |
| 359 |
for i in range(10): |
| 360 |
pile.spawn(passthru, i + unique) |
| 361 |
|
| 362 |
eventlet.spawn(bunch_of_work, pile1, 0) |
| 363 |
eventlet.spawn(bunch_of_work, pile2, 100) |
| 364 |
eventlet.sleep(0) |
| 365 |
self.assertEqual(list(pile2), list(range(100, 110))) |
| 366 |
self.assertEqual(list(pile1), list(range(10))) |
| 367 |
|
| 368 |
|
| 369 |
def test_greenpool_type_check(): |
| 370 |
eventlet.GreenPool(0) |
| 371 |
eventlet.GreenPool(1) |
| 372 |
eventlet.GreenPool(1e3) |
| 373 |
|
| 374 |
with tests.assert_raises(TypeError): |
| 375 |
eventlet.GreenPool('foo') |
| 376 |
with tests.assert_raises(ValueError): |
| 377 |
eventlet.GreenPool(-1) |
| 378 |
|
| 379 |
|
| 380 |
class StressException(Exception): |
| 381 |
pass |
| 382 |
|
| 383 |
|
| 384 |
r = random.Random(0) |
| 385 |
|
| 386 |
|
| 387 |
def pressure(arg): |
| 388 |
while r.random() < 0.5: |
| 389 |
eventlet.sleep(r.random() * 0.001) |
| 390 |
if r.random() < 0.8: |
| 391 |
return arg |
| 392 |
else: |
| 393 |
raise StressException(arg) |
| 394 |
|
| 395 |
|
| 396 |
def passthru(arg): |
| 397 |
while r.random() < 0.5: |
| 398 |
eventlet.sleep(r.random() * 0.001) |
| 399 |
return arg |
| 400 |
|
| 401 |
|
| 402 |
class Stress(tests.LimitedTestCase): |
| 403 |
# tests will take extra-long |
| 404 |
TEST_TIMEOUT = 60 |
| 405 |
|
| 406 |
def spawn_order_check(self, concurrency): |
| 407 |
# checks that piles are strictly ordered |
| 408 |
p = eventlet.GreenPile(concurrency) |
| 409 |
|
| 410 |
def makework(count, unique): |
| 411 |
for i in range(count): |
| 412 |
token = (unique, i) |
| 413 |
p.spawn(pressure, token) |
| 414 |
|
| 415 |
iters = 1000 |
| 416 |
eventlet.spawn(makework, iters, 1) |
| 417 |
eventlet.spawn(makework, iters, 2) |
| 418 |
eventlet.spawn(makework, iters, 3) |
| 419 |
p.spawn(pressure, (0, 0)) |
| 420 |
latest = [-1] * 4 |
| 421 |
received = 0 |
| 422 |
it = iter(p) |
| 423 |
while True: |
| 424 |
try: |
| 425 |
i = next(it) |
| 426 |
except StressException as exc: |
| 427 |
i = exc.args[0] |
| 428 |
except StopIteration: |
| 429 |
break |
| 430 |
received += 1 |
| 431 |
if received % 5 == 0: |
| 432 |
eventlet.sleep(0.0001) |
| 433 |
unique, order = i |
| 434 |
assert latest[unique] < order |
| 435 |
latest[unique] = order |
| 436 |
for l in latest[1:]: |
| 437 |
self.assertEqual(l, iters - 1) |
| 438 |
|
| 439 |
def test_ordering_5(self): |
| 440 |
self.spawn_order_check(5) |
| 441 |
|
| 442 |
def test_ordering_50(self): |
| 443 |
self.spawn_order_check(50) |
| 444 |
|
| 445 |
def imap_memory_check(self, concurrency): |
| 446 |
# checks that imap is strictly |
| 447 |
# ordered and consumes a constant amount of memory |
| 448 |
p = eventlet.GreenPool(concurrency) |
| 449 |
count = 1000 |
| 450 |
it = p.imap(passthru, range(count)) |
| 451 |
latest = -1 |
| 452 |
while True: |
| 453 |
try: |
| 454 |
i = next(it) |
| 455 |
except StopIteration: |
| 456 |
break |
| 457 |
|
| 458 |
if latest == -1: |
| 459 |
gc.collect() |
| 460 |
initial_obj_count = len(gc.get_objects()) |
| 461 |
assert i > latest |
| 462 |
latest = i |
| 463 |
if latest % 5 == 0: |
| 464 |
eventlet.sleep(0.001) |
| 465 |
if latest % 10 == 0: |
| 466 |
gc.collect() |
| 467 |
objs_created = len(gc.get_objects()) - initial_obj_count |
| 468 |
assert objs_created < 25 * concurrency, objs_created |
| 469 |
# make sure we got to the end |
| 470 |
self.assertEqual(latest, count - 1) |
| 471 |
|
| 472 |
def test_imap_50(self): |
| 473 |
self.imap_memory_check(50) |
| 474 |
|
| 475 |
def test_imap_500(self): |
| 476 |
self.imap_memory_check(500) |
| 477 |
|
| 478 |
def test_with_intpool(self): |
| 479 |
class IntPool(pools.Pool): |
| 480 |
def create(self): |
| 481 |
self.current_integer = getattr(self, 'current_integer', 0) + 1 |
| 482 |
return self.current_integer |
| 483 |
|
| 484 |
def subtest(intpool_size, pool_size, num_executes): |
| 485 |
def run(int_pool): |
| 486 |
token = int_pool.get() |
| 487 |
eventlet.sleep(0.0001) |
| 488 |
int_pool.put(token) |
| 489 |
return token |
| 490 |
|
| 491 |
int_pool = IntPool(max_size=intpool_size) |
| 492 |
pool = eventlet.GreenPool(pool_size) |
| 493 |
for ix in range(num_executes): |
| 494 |
pool.spawn(run, int_pool) |
| 495 |
pool.waitall() |
| 496 |
|
| 497 |
subtest(4, 7, 7) |
| 498 |
subtest(50, 75, 100) |
| 499 |
for isize in (10, 20, 30, 40, 50): |
| 500 |
for psize in (5, 25, 35, 50): |
| 501 |
subtest(isize, psize, psize) |