Debian Codegraph @

tests/db_pool_test.py

raw
1 import os
2 import sys
3 import traceback
4
5 from eventlet import db_pool
6 import eventlet
7 import eventlet.tpool
8 import tests
9 import tests.mock
10
11 psycopg2 = None
12 try:
13 import psycopg2
14 import psycopg2.extensions
15 except ImportError:
16 pass
17
18 MySQLdb = None
19 try:
20 import MySQLdb
21 except ImportError:
22 pass
23
24
25 class DBTester:
26 __test__ = False # so that nose doesn't try to execute this directly
27
28 def setUp(self):
29 self.create_db()
30 self.connection = None
31 connection = self._dbmodule.connect(**self._auth)
32 cursor = connection.cursor()
33 cursor.execute("""CREATE TABLE gargleblatz
34 (
35 a INTEGER
36 );""")
37 connection.commit()
38 cursor.close()
39 connection.close()
40
41 def tearDown(self):
42 if self.connection:
43 self.connection.close()
44 self.drop_db()
45
46 def set_up_dummy_table(self, connection=None):
47 close_connection = False
48 if connection is None:
49 close_connection = True
50 if self.connection is None:
51 connection = self._dbmodule.connect(**self._auth)
52 else:
53 connection = self.connection
54
55 cursor = connection.cursor()
56 cursor.execute(self.dummy_table_sql)
57 connection.commit()
58 cursor.close()
59 if close_connection:
60 connection.close()
61
62
63 # silly mock class
64 class Mock:
65 pass
66
67
68 class DBConnectionPool(DBTester):
69 __test__ = False # so that nose doesn't try to execute this directly
70
71 def setUp(self):
72 super().setUp()
73 self.pool = self.create_pool()
74 self.connection = self.pool.get()
75
76 def tearDown(self):
77 if self.connection:
78 self.pool.put(self.connection)
79 self.pool.clear()
80 super().tearDown()
81
82 def assert_cursor_works(self, cursor):
83 cursor.execute("select 1")
84 rows = cursor.fetchall()
85 assert rows
86
87 def test_connecting(self):
88 assert self.connection is not None
89
90 def test_create_cursor(self):
91 cursor = self.connection.cursor()
92 cursor.close()
93
94 def test_run_query(self):
95 cursor = self.connection.cursor()
96 self.assert_cursor_works(cursor)
97 cursor.close()
98
99 def test_run_bad_query(self):
100 cursor = self.connection.cursor()
101 try:
102 cursor.execute("garbage blah blah")
103 assert False
104 except AssertionError:
105 raise
106 except Exception:
107 pass
108 cursor.close()
109
110 def test_put_none(self):
111 # the pool is of size 1, and its only connection is out
112 assert self.pool.free() == 0
113 self.pool.put(None)
114 # ha ha we fooled it into thinking that we had a dead process
115 assert self.pool.free() == 1
116 conn2 = self.pool.get()
117 assert conn2 is not None
118 assert conn2.cursor
119 self.pool.put(conn2)
120
121 def test_close_does_a_put(self):
122 assert self.pool.free() == 0
123 self.connection.close()
124 assert self.pool.free() == 1
125 self.assertRaises(AttributeError, self.connection.cursor)
126
127 def test_put_doesnt_double_wrap(self):
128 self.pool.put(self.connection)
129 conn = self.pool.get()
130 assert not isinstance(conn._base, db_pool.PooledConnectionWrapper)
131 self.pool.put(conn)
132
133 def test_bool(self):
134 assert self.connection
135 self.connection.close()
136 assert not self.connection
137
138 def fill_up_table(self, conn):
139 curs = conn.cursor()
140 for i in range(1000):
141 curs.execute('insert into test_table (value_int) values (%s)' % i)
142 conn.commit()
143
144 def test_returns_immediately(self):
145 self.pool = self.create_pool()
146 conn = self.pool.get()
147 self.set_up_dummy_table(conn)
148 self.fill_up_table(conn)
149 curs = conn.cursor()
150 results = []
151 SHORT_QUERY = "select * from test_table"
152 evt = eventlet.Event()
153
154 def a_query():
155 self.assert_cursor_works(curs)
156 curs.execute(SHORT_QUERY)
157 results.append(2)
158 evt.send()
159 eventlet.spawn(a_query)
160 results.append(1)
161 self.assertEqual([1], results)
162 evt.wait()
163 self.assertEqual([1, 2], results)
164 self.pool.put(conn)
165
166 def test_connection_is_clean_after_put(self):
167 self.pool = self.create_pool()
168 conn = self.pool.get()
169 self.set_up_dummy_table(conn)
170 curs = conn.cursor()
171 for i in range(10):
172 curs.execute('insert into test_table (value_int) values (%s)' % i)
173 # do not commit :-)
174 self.pool.put(conn)
175 del conn
176 conn2 = self.pool.get()
177 curs2 = conn2.cursor()
178 for i in range(10):
179 curs2.execute('insert into test_table (value_int) values (%s)' % i)
180 conn2.commit()
181 curs2.execute("select * from test_table")
182 # we should have only inserted them once
183 self.assertEqual(10, curs2.rowcount)
184 self.pool.put(conn2)
185
186 def test_visibility_from_other_connections(self):
187 self.pool = self.create_pool(max_size=3)
188 conn = self.pool.get()
189 conn2 = self.pool.get()
190 curs = conn.cursor()
191 try:
192 curs2 = conn2.cursor()
193 curs2.execute("insert into gargleblatz (a) values (%s)" % (314159))
194 self.assertEqual(curs2.rowcount, 1)
195 conn2.commit()
196 selection_query = "select * from gargleblatz"
197 curs2.execute(selection_query)
198 self.assertEqual(curs2.rowcount, 1)
199 del curs2
200 self.pool.put(conn2)
201 # create a new connection, it should see the addition
202 conn3 = self.pool.get()
203 curs3 = conn3.cursor()
204 curs3.execute(selection_query)
205 self.assertEqual(curs3.rowcount, 1)
206 # now, does the already-open connection see it?
207 curs.execute(selection_query)
208 self.assertEqual(curs.rowcount, 1)
209 self.pool.put(conn3)
210 finally:
211 # clean up my litter
212 curs.execute("delete from gargleblatz where a=314159")
213 conn.commit()
214 self.pool.put(conn)
215
216 def test_clear(self):
217 self.pool = self.create_pool()
218 self.pool.put(self.connection)
219 self.pool.clear()
220 self.assertEqual(len(self.pool.free_items), 0)
221
222 def test_clear_warmup(self):
223 """Clear implicitly created connections (min_size > 0)"""
224 self.pool = self.create_pool(min_size=1)
225 self.pool.clear()
226 self.assertEqual(len(self.pool.free_items), 0)
227
228 def test_unwrap_connection(self):
229 self.assertTrue(isinstance(self.connection,
230 db_pool.GenericConnectionWrapper))
231 conn = self.pool._unwrap_connection(self.connection)
232 assert not isinstance(conn, db_pool.GenericConnectionWrapper)
233
234 self.assertEqual(None, self.pool._unwrap_connection(None))
235 self.assertEqual(None, self.pool._unwrap_connection(1))
236
237 # testing duck typing here -- as long as the connection has a
238 # _base attribute, it should be unwrappable
239 x = Mock()
240 x._base = 'hi'
241 self.assertEqual('hi', self.pool._unwrap_connection(x))
242 conn.close()
243
244 def test_safe_close(self):
245 self.pool._safe_close(self.connection, quiet=True)
246 self.assertEqual(len(self.pool.free_items), 1)
247
248 self.pool._safe_close(None)
249 self.pool._safe_close(1)
250
251 # now we're really going for 100% coverage
252 x = Mock()
253
254 def fail():
255 raise KeyboardInterrupt()
256 x.close = fail
257 self.assertRaises(KeyboardInterrupt, self.pool._safe_close, x)
258
259 x = Mock()
260
261 def fail2():
262 raise RuntimeError("if this line has been printed, the test succeeded")
263 x.close = fail2
264 self.pool._safe_close(x, quiet=False)
265
266 def test_zero_max_idle(self):
267 self.pool.put(self.connection)
268 self.pool.clear()
269 self.pool = self.create_pool(max_size=2, max_idle=0)
270 self.connection = self.pool.get()
271 self.connection.close()
272 self.assertEqual(len(self.pool.free_items), 0)
273
274 def test_zero_max_age(self):
275 self.pool.put(self.connection)
276 self.pool.clear()
277 self.pool = self.create_pool(max_size=2, max_age=0)
278 self.connection = self.pool.get()
279 self.connection.close()
280 self.assertEqual(len(self.pool.free_items), 0)
281
282 def test_waiters_get_woken(self):
283 # verify that when there's someone waiting on an empty pool
284 # and someone puts an immediately-closed connection back in
285 # the pool that the waiter gets woken
286 self.pool.put(self.connection)
287 self.pool.clear()
288 self.pool = self.create_pool(max_size=1, max_age=0)
289
290 self.connection = self.pool.get()
291 self.assertEqual(self.pool.free(), 0)
292 self.assertEqual(self.pool.waiting(), 0)
293 e = eventlet.Event()
294
295 def retrieve(pool, ev):
296 c = pool.get()
297 ev.send(c)
298 eventlet.spawn(retrieve, self.pool, e)
299 eventlet.sleep(0) # these two sleeps should advance the retrieve
300 eventlet.sleep(0) # coroutine until it's waiting in get()
301 self.assertEqual(self.pool.free(), 0)
302 self.assertEqual(self.pool.waiting(), 1)
303 self.pool.put(self.connection)
304 timer = eventlet.Timeout(1)
305 conn = e.wait()
306 timer.cancel()
307 self.assertEqual(self.pool.free(), 0)
308 self.assertEqual(self.pool.waiting(), 0)
309 self.pool.put(conn)
310
311 def test_raising_create(self):
312 # if the create() method raises an exception the pool should
313 # not lose any connections
314 self.pool = self.create_pool(max_size=1, module=RaisingDBModule())
315 self.assertRaises(RuntimeError, self.pool.get)
316 self.assertEqual(self.pool.free(), 1)
317
318
319 class DummyConnection:
320 def rollback(self):
321 pass
322
323
324 class DummyDBModule:
325 def connect(self, *args, **kwargs):
326 return DummyConnection()
327
328
329 class RaisingDBModule:
330 def connect(self, *args, **kw):
331 raise RuntimeError()
332
333
334 class TpoolConnectionPool(DBConnectionPool):
335 __test__ = False # so that nose doesn't try to execute this directly
336
337 def create_pool(self, min_size=0, max_size=1, max_idle=10, max_age=10,
338 connect_timeout=0.5, module=None):
339 if module is None:
340 module = self._dbmodule
341 return db_pool.TpooledConnectionPool(
342 module,
343 min_size=min_size, max_size=max_size,
344 max_idle=max_idle, max_age=max_age,
345 connect_timeout=connect_timeout,
346 **self._auth)
347
348 def setUp(self):
349 super().setUp()
350
351 def tearDown(self):
352 super().tearDown()
353 eventlet.tpool.killall()
354
355
356 class RawConnectionPool(DBConnectionPool):
357 __test__ = False # so that nose doesn't try to execute this directly
358
359 def create_pool(self, min_size=0, max_size=1, max_idle=10, max_age=10,
360 connect_timeout=0.5, module=None):
361 if module is None:
362 module = self._dbmodule
363 return db_pool.RawConnectionPool(
364 module,
365 min_size=min_size, max_size=max_size,
366 max_idle=max_idle, max_age=max_age,
367 connect_timeout=connect_timeout,
368 **self._auth)
369
370
371 def test_raw_pool_issue_125():
372 # pool = self.create_pool(min_size=3, max_size=5)
373 pool = db_pool.RawConnectionPool(
374 DummyDBModule(),
375 dsn="dbname=test user=jessica port=5433",
376 min_size=3, max_size=5)
377 conn = pool.get()
378 pool.put(conn)
379
380
381 def test_raw_pool_custom_cleanup_ok():
382 cleanup_mock = tests.mock.Mock()
383 pool = db_pool.RawConnectionPool(DummyDBModule(), cleanup=cleanup_mock)
384 conn = pool.get()
385 pool.put(conn)
386 assert cleanup_mock.call_count == 1
387
388 with pool.item() as conn:
389 pass
390 assert cleanup_mock.call_count == 2
391
392
393 def test_raw_pool_custom_cleanup_arg_error():
394 cleanup_mock = tests.mock.Mock(side_effect=NotImplementedError)
395 pool = db_pool.RawConnectionPool(DummyDBModule())
396 conn = pool.get()
397 pool.put(conn, cleanup=cleanup_mock)
398 assert cleanup_mock.call_count == 1
399
400 with pool.item(cleanup=cleanup_mock):
401 pass
402 assert cleanup_mock.call_count == 2
403
404
405 def test_raw_pool_custom_cleanup_fatal():
406 state = [0]
407
408 def cleanup(conn):
409 state[0] += 1
410 raise KeyboardInterrupt
411
412 pool = db_pool.RawConnectionPool(DummyDBModule(), cleanup=cleanup)
413 conn = pool.get()
414 try:
415 pool.put(conn)
416 except KeyboardInterrupt:
417 pass
418 else:
419 assert False, 'Expected KeyboardInterrupt'
420 assert state[0] == 1
421
422
423 def test_raw_pool_clear_update_current_size():
424 # https://github.com/eventlet/eventlet/issues/139
425 # BaseConnectionPool.clear does not update .current_size.
426 # That leads to situation when new connections could not be created.
427 pool = db_pool.RawConnectionPool(DummyDBModule())
428 pool.get().close()
429 assert pool.current_size == 1
430 assert len(pool.free_items) == 1
431 pool.clear()
432 assert pool.current_size == 0
433 assert len(pool.free_items) == 0
434
435
436 def mysql_requirement(_f):
437 verbose = os.environ.get('eventlet_test_mysql_verbose')
438 if MySQLdb is None:
439 if verbose:
440 print(">> Skipping mysql tests, MySQLdb not importable", file=sys.stderr)
441 return False
442
443 try:
444 auth = tests.get_database_auth()['MySQLdb'].copy()
445 MySQLdb.connect(**auth)
446 return True
447 except MySQLdb.OperationalError:
448 if verbose:
449 print(">> Skipping mysql tests, error when connecting:", file=sys.stderr)
450 traceback.print_exc()
451 return False
452
453
454 class MysqlConnectionPool:
455 dummy_table_sql = """CREATE TEMPORARY TABLE test_table
456 (
457 row_id INTEGER PRIMARY KEY AUTO_INCREMENT,
458 value_int INTEGER,
459 value_float FLOAT,
460 value_string VARCHAR(200),
461 value_uuid CHAR(36),
462 value_binary BLOB,
463 value_binary_string VARCHAR(200) BINARY,
464 value_enum ENUM('Y','N'),
465 created TIMESTAMP
466 ) ENGINE=InnoDB;"""
467
468 @tests.skip_unless(mysql_requirement)
469 def setUp(self):
470 self._dbmodule = MySQLdb
471 self._auth = tests.get_database_auth()['MySQLdb']
472 super().setUp()
473
474 def tearDown(self):
475 super().tearDown()
476
477 def create_db(self):
478 auth = self._auth.copy()
479 try:
480 self.drop_db()
481 except Exception:
482 pass
483 dbname = 'test%s' % os.getpid()
484 db = self._dbmodule.connect(**auth).cursor()
485 db.execute("create database " + dbname)
486 db.close()
487 self._auth['db'] = dbname
488 del db
489
490 def drop_db(self):
491 db = self._dbmodule.connect(**self._auth).cursor()
492 db.execute("drop database IF EXISTS " + self._auth['db'])
493 db.close()
494 del db
495
496
497 class Test01MysqlTpool(MysqlConnectionPool, TpoolConnectionPool, tests.LimitedTestCase):
498 __test__ = True
499
500
501 class Test02MysqlRaw(MysqlConnectionPool, RawConnectionPool, tests.LimitedTestCase):
502 __test__ = True
503
504
505 def postgres_requirement(_f):
506 if psycopg2 is None:
507 print("Skipping postgres tests, psycopg2 not importable")
508 return False
509
510 try:
511 auth = tests.get_database_auth()['psycopg2'].copy()
512 psycopg2.connect(**auth)
513 return True
514 except psycopg2.OperationalError:
515 print("Skipping postgres tests, error when connecting")
516 return False
517
518
519 class Psycopg2ConnectionPool:
520 dummy_table_sql = """CREATE TEMPORARY TABLE test_table
521 (
522 row_id SERIAL PRIMARY KEY,
523 value_int INTEGER,
524 value_float FLOAT,
525 value_string VARCHAR(200),
526 value_uuid CHAR(36),
527 value_binary BYTEA,
528 value_binary_string BYTEA,
529 created TIMESTAMP
530 );"""
531
532 @tests.skip_unless(postgres_requirement)
533 def setUp(self):
534 self._dbmodule = psycopg2
535 self._auth = tests.get_database_auth()['psycopg2']
536 super().setUp()
537
538 def tearDown(self):
539 super().tearDown()
540
541 def create_db(self):
542 dbname = 'test%s' % os.getpid()
543 self._auth['database'] = dbname
544 try:
545 self.drop_db()
546 except Exception:
547 pass
548 auth = self._auth.copy()
549 auth.pop('database') # can't create if you're connecting to it
550 conn = self._dbmodule.connect(**auth)
551 conn.set_isolation_level(0)
552 db = conn.cursor()
553 db.execute("create database " + dbname)
554 db.close()
555 conn.close()
556
557 def drop_db(self):
558 auth = self._auth.copy()
559 auth.pop('database') # can't drop database we connected to
560 conn = self._dbmodule.connect(**auth)
561 conn.set_isolation_level(0)
562 db = conn.cursor()
563 db.execute("drop database IF EXISTS " + self._auth['database'])
564 db.close()
565 conn.close()
566
567
568 class TestPsycopg2Base(tests.LimitedTestCase):
569 __test__ = False
570
571 def test_cursor_works_as_context_manager(self):
572 with self.connection.cursor() as c:
573 c.execute('select 1')
574 row = c.fetchone()
575 assert row == (1,)
576
577 def test_set_isolation_level(self):
578 self.connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
579
580
581 class Test01Psycopg2Tpool(Psycopg2ConnectionPool, TpoolConnectionPool, TestPsycopg2Base):
582 __test__ = True
583
584
585 class Test02Psycopg2Raw(Psycopg2ConnectionPool, RawConnectionPool, TestPsycopg2Base):
586 __test__ = True