Debian Codegraph @

taskflow/tests/unit/persistence/test_sql_persistence.py

raw
1 # Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import abc
16 import contextlib
17 import os
18 import random
19 import tempfile
20
21 import testtools
22
23
24 # NOTE(harlowja): by default this will test against sqlite using a temporary
25 # sqlite file (this is done instead of in-memory to ensure thread safety,
26 # in-memory sqlite is not thread safe).
27 #
28 # There are also "opportunistic" tests for both mysql and postgresql in here,
29 # which allows testing against all 3 databases (sqlite, mysql, postgres) in
30 # a properly configured unit test environment. For the opportunistic testing
31 # you need to set up a db user 'openstack_citest' with password
32 # 'openstack_citest' that has the permissions to create databases on
33 # localhost.
34
35 USER = "openstack_citest"
36 PASSWD = "openstack_citest"
37 DATABASE = "tftest_" + ''.join(random.choice('0123456789')
38 for _ in range(12))
39
40 import sqlalchemy as sa
41
42 from taskflow.persistence import backends
43 from taskflow.persistence.backends import impl_sqlalchemy
44 from taskflow import test
45 from taskflow.tests.unit.persistence import base
46
47
48 def _get_connect_string(backend, user, passwd, database=None, variant=None):
49 """Forms a sqlalchemy database uri string for the given values."""
50 if backend == "postgres":
51 if not variant:
52 variant = 'psycopg2'
53 backend = "postgresql+%s" % (variant)
54 elif backend == "mysql":
55 if not variant:
56 variant = 'pymysql'
57 backend = "mysql+%s" % (variant)
58 else:
59 raise Exception("Unrecognized backend: '%s'" % backend)
60 if not database:
61 database = ''
62 return f"{backend}://{user}:{passwd}@localhost/{database}"
63
64
65 def _mysql_exists():
66 engine = None
67 try:
68 db_uri = _get_connect_string('mysql', USER, PASSWD)
69 engine = sa.create_engine(db_uri)
70 with contextlib.closing(engine.connect()):
71 return True
72 except Exception:
73 pass
74 finally:
75 if engine is not None:
76 try:
77 engine.dispose()
78 except Exception:
79 pass
80 return False
81
82
83 def _postgres_exists():
84 engine = None
85 try:
86 db_uri = _get_connect_string('postgres', USER, PASSWD, 'postgres')
87 engine = sa.create_engine(db_uri)
88 with contextlib.closing(engine.connect()):
89 return True
90 except Exception:
91 return False
92 finally:
93 if engine is not None:
94 try:
95 engine.dispose()
96 except Exception:
97 pass
98
99
100 class SqlitePersistenceTest(test.TestCase, base.PersistenceTestMixin):
101 """Inherits from the base test and sets up a sqlite temporary db."""
102 def _get_connection(self):
103 conf = {
104 'connection': self.db_uri,
105 }
106 return impl_sqlalchemy.SQLAlchemyBackend(conf).get_connection()
107
108 def setUp(self):
109 super().setUp()
110 self.db_location = tempfile.mktemp(suffix='.db')
111 self.db_uri = "sqlite:///%s" % (self.db_location)
112 # Ensure upgraded to the right schema
113 with contextlib.closing(self._get_connection()) as conn:
114 conn.upgrade()
115
116 def tearDown(self):
117 super().tearDown()
118 if self.db_location and os.path.isfile(self.db_location):
119 os.unlink(self.db_location)
120 self.db_location = None
121
122
123 class BackendPersistenceTestMixin(base.PersistenceTestMixin,
124 metaclass=abc.ABCMeta):
125 """Specifies a backend type and does required setup and teardown."""
126
127 def _get_connection(self):
128 return self.backend.get_connection()
129
130 def test_entrypoint(self):
131 # Test that the entrypoint fetching also works (even with dialects)
132 # using the same configuration we used in setUp() but not using
133 # the impl_sqlalchemy SQLAlchemyBackend class directly...
134 with contextlib.closing(backends.fetch(self.db_conf)) as backend:
135 with contextlib.closing(backend.get_connection()):
136 pass
137
138 @abc.abstractmethod
139 def _init_db(self):
140 """Sets up the database, and returns the uri to that database."""
141
142 @abc.abstractmethod
143 def _remove_db(self):
144 """Cleans up by removing the database once the tests are done."""
145
146 def setUp(self):
147 super().setUp()
148 self.backend = None
149 try:
150 self.db_uri = self._init_db()
151 self.db_conf = {
152 'connection': self.db_uri
153 }
154 # Since we are using random database names, we need to make sure
155 # and remove our random database when we are done testing.
156 self.addCleanup(self._remove_db)
157 except Exception as e:
158 self.skipTest("Failed to create temporary database;"
159 " testing being skipped due to: %s" % (e))
160 else:
161 self.backend = impl_sqlalchemy.SQLAlchemyBackend(self.db_conf)
162 self.addCleanup(self.backend.close)
163 with contextlib.closing(self._get_connection()) as conn:
164 conn.upgrade()
165
166
167 @testtools.skipIf(not _mysql_exists(), 'mysql is not available')
168 class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
169
170 def _init_db(self):
171 engine = None
172 try:
173 db_uri = _get_connect_string('mysql', USER, PASSWD)
174 engine = sa.create_engine(db_uri)
175 with contextlib.closing(engine.connect()) as conn:
176 conn.execute(sa.text("CREATE DATABASE %s" % DATABASE))
177 except Exception as e:
178 raise Exception('Failed to initialize MySQL db: %s' % (e))
179 finally:
180 if engine is not None:
181 try:
182 engine.dispose()
183 except Exception:
184 pass
185 return _get_connect_string('mysql', USER, PASSWD,
186 database=DATABASE)
187
188 def _remove_db(self):
189 engine = None
190 try:
191 engine = sa.create_engine(self.db_uri)
192 with contextlib.closing(engine.connect()) as conn:
193 conn.execute(sa.text("DROP DATABASE IF EXISTS %s" % DATABASE))
194 except Exception as e:
195 raise Exception('Failed to remove temporary database: %s' % (e))
196 finally:
197 if engine is not None:
198 try:
199 engine.dispose()
200 except Exception:
201 pass
202
203
204 @testtools.skipIf(not _postgres_exists(), 'postgres is not available')
205 class PostgresPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
206
207 def _init_db(self):
208 engine = None
209 try:
210 # Postgres can't operate on the database it's connected to, that's
211 # why we connect to the database 'postgres' and then create the
212 # desired database.
213 db_uri = _get_connect_string('postgres', USER, PASSWD,
214 database='postgres')
215 engine = sa.create_engine(db_uri)
216 with contextlib.closing(engine.connect()) as conn:
217 conn.connection.set_isolation_level(0)
218 conn.execute(sa.text("CREATE DATABASE %s" % DATABASE))
219 conn.connection.set_isolation_level(1)
220 except Exception as e:
221 raise Exception('Failed to initialize PostgreSQL db: %s' % (e))
222 finally:
223 if engine is not None:
224 try:
225 engine.dispose()
226 except Exception:
227 pass
228 return _get_connect_string('postgres', USER, PASSWD,
229 database=DATABASE)
230
231 def _remove_db(self):
232 engine = None
233 try:
234 # Postgres can't operate on the database it's connected to, that's
235 # why we connect to the 'postgres' database and then drop the
236 # database.
237 db_uri = _get_connect_string('postgres', USER, PASSWD,
238 database='postgres')
239 engine = sa.create_engine(db_uri)
240 with contextlib.closing(engine.connect()) as conn:
241 conn.connection.set_isolation_level(0)
242 conn.execute(sa.text("DROP DATABASE IF EXISTS %s" % DATABASE))
243 conn.connection.set_isolation_level(1)
244 except Exception as e:
245 raise Exception('Failed to remove temporary database: %s' % (e))
246 finally:
247 if engine is not None:
248 try:
249 engine.dispose()
250 except Exception:
251 pass
252
253
254 class SQLBackendFetchingTest(test.TestCase):
255
256 def test_sqlite_persistence_entry_point(self):
257 conf = {'connection': 'sqlite:///'}
258 with contextlib.closing(backends.fetch(conf)) as be:
259 self.assertIsInstance(be, impl_sqlalchemy.SQLAlchemyBackend)
260
261 @testtools.skipIf(not _mysql_exists(), 'mysql is not available')
262 def test_mysql_persistence_entry_point(self):
263 uri = _get_connect_string('mysql', USER, PASSWD, database=DATABASE)
264 conf = {'connection': uri}
265 with contextlib.closing(backends.fetch(conf)) as be:
266 self.assertIsInstance(be, impl_sqlalchemy.SQLAlchemyBackend)
267
268 @testtools.skipIf(not _postgres_exists(), 'postgres is not available')
269 def test_postgres_persistence_entry_point(self):
270 uri = _get_connect_string('postgres', USER, PASSWD, database=DATABASE)
271 conf = {'connection': uri}
272 with contextlib.closing(backends.fetch(conf)) as be:
273 self.assertIsInstance(be, impl_sqlalchemy.SQLAlchemyBackend)