Debian Codegraph @

django/db/backends/base/creation.py

raw
1 import os
2 import sys
3 from io import StringIO
4
5 from django.apps import apps
6 from django.conf import settings
7 from django.core import serializers
8 from django.db import router
9 from django.db.transaction import atomic
10 from django.utils.module_loading import import_string
11
12 # The prefix to put on the default database name when creating
13 # the test database.
14 TEST_DATABASE_PREFIX = "test_"
15
16
17 class BaseDatabaseCreation:
18 """
19 Encapsulate backend-specific differences pertaining to creation and
20 destruction of the test database.
21 """
22
23 def __init__(self, connection):
24 self.connection = connection
25
26 def _nodb_cursor(self):
27 return self.connection._nodb_cursor()
28
29 def log(self, msg):
30 sys.stderr.write(msg + os.linesep)
31
32 def create_test_db(
33 self, verbosity=1, autoclobber=False, serialize=True, keepdb=False
34 ):
35 """
36 Create a test database, prompting the user for confirmation if the
37 database already exists. Return the name of the test database created.
38 """
39 # Don't import django.core.management if it isn't needed.
40 from django.core.management import call_command
41
42 test_database_name = self._get_test_db_name()
43
44 if verbosity >= 1:
45 action = "Creating"
46 if keepdb:
47 action = "Using existing"
48
49 self.log(
50 "%s test database for alias %s..."
51 % (
52 action,
53 self._get_database_display_str(verbosity, test_database_name),
54 )
55 )
56
57 # We could skip this call if keepdb is True, but we instead
58 # give it the keepdb param. This is to handle the case
59 # where the test DB doesn't exist, in which case we need to
60 # create it, then just not destroy it. If we instead skip
61 # this, we will get an exception.
62 self._create_test_db(verbosity, autoclobber, keepdb)
63
64 self.connection.close()
65 settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
66 self.connection.settings_dict["NAME"] = test_database_name
67
68 try:
69 if self.connection.settings_dict["TEST"]["MIGRATE"] is False:
70 # Disable migrations for all apps.
71 old_migration_modules = settings.MIGRATION_MODULES
72 settings.MIGRATION_MODULES = {
73 app.label: None for app in apps.get_app_configs()
74 }
75 # We report migrate messages at one level lower than that
76 # requested. This ensures we don't get flooded with messages during
77 # testing (unless you really ask to be flooded).
78 call_command(
79 "migrate",
80 verbosity=max(verbosity - 1, 0),
81 interactive=False,
82 database=self.connection.alias,
83 run_syncdb=True,
84 )
85 finally:
86 if self.connection.settings_dict["TEST"]["MIGRATE"] is False:
87 settings.MIGRATION_MODULES = old_migration_modules
88
89 # We then serialize the current state of the database into a string
90 # and store it on the connection. This slightly horrific process is so people
91 # who are testing on databases without transactions or who are using
92 # a TransactionTestCase still get a clean database on every test run.
93 if serialize:
94 self.connection._test_serialized_contents = self.serialize_db_to_string()
95
96 call_command("createcachetable", database=self.connection.alias)
97
98 # Ensure a connection for the side effect of initializing the test database.
99 self.connection.ensure_connection()
100
101 if os.environ.get("RUNNING_DJANGOS_TEST_SUITE") == "true":
102 self.mark_expected_failures_and_skips()
103
104 return test_database_name
105
106 def set_as_test_mirror(self, primary_settings_dict):
107 """
108 Set this database up to be used in testing as a mirror of a primary
109 database whose settings are given.
110 """
111 self.connection.settings_dict["NAME"] = primary_settings_dict["NAME"]
112
113 def serialize_db_to_string(self):
114 """
115 Serialize all data in the database into a JSON string.
116 Designed only for test runner usage; will not handle large
117 amounts of data.
118 """
119
120 # Iteratively return every object for all models to serialize.
121 def get_objects():
122 from django.db.migrations.loader import MigrationLoader
123
124 loader = MigrationLoader(self.connection)
125 for app_config in apps.get_app_configs():
126 if (
127 app_config.models_module is not None
128 and app_config.label in loader.migrated_apps
129 and app_config.name not in settings.TEST_NON_SERIALIZED_APPS
130 ):
131 for model in app_config.get_models():
132 if model._meta.can_migrate(
133 self.connection
134 ) and router.allow_migrate_model(self.connection.alias, model):
135 queryset = model._base_manager.using(
136 self.connection.alias,
137 ).order_by(model._meta.pk.name)
138 chunk_size = (
139 2000 if queryset._prefetch_related_lookups else None
140 )
141 yield from queryset.iterator(chunk_size=chunk_size)
142
143 # Serialize to a string
144 out = StringIO()
145 serializers.serialize("json", get_objects(), indent=None, stream=out)
146 return out.getvalue()
147
148 def deserialize_db_from_string(self, data):
149 """
150 Reload the database with data from a string generated by
151 the serialize_db_to_string() method.
152 """
153 data = StringIO(data)
154 table_names = set()
155 # Load data in a transaction to handle forward references and cycles.
156 with atomic(using=self.connection.alias):
157 # Disable constraint checks, because some databases (MySQL) doesn't
158 # support deferred checks.
159 with self.connection.constraint_checks_disabled():
160 for obj in serializers.deserialize(
161 "json", data, using=self.connection.alias
162 ):
163 obj.save()
164 table_names.add(obj.object.__class__._meta.db_table)
165 # Manually check for any invalid keys that might have been added,
166 # because constraint checks were disabled.
167 self.connection.check_constraints(table_names=table_names)
168
169 def _get_database_display_str(self, verbosity, database_name):
170 """
171 Return display string for a database for use in various actions.
172 """
173 return "'%s'%s" % (
174 self.connection.alias,
175 (" ('%s')" % database_name) if verbosity >= 2 else "",
176 )
177
178 def _get_test_db_name(self):
179 """
180 Internal implementation - return the name of the test DB that will be
181 created. Only useful when called from create_test_db() and
182 _create_test_db() and when no external munging is done with the 'NAME'
183 settings.
184 """
185 if self.connection.settings_dict["TEST"]["NAME"]:
186 return self.connection.settings_dict["TEST"]["NAME"]
187 return TEST_DATABASE_PREFIX + self.connection.settings_dict["NAME"]
188
189 def _execute_create_test_db(self, cursor, parameters, keepdb=False):
190 cursor.execute("CREATE DATABASE %(dbname)s %(suffix)s" % parameters)
191
192 def _create_test_db(self, verbosity, autoclobber, keepdb=False):
193 """
194 Internal implementation - create the test db tables.
195 """
196 test_database_name = self._get_test_db_name()
197 test_db_params = {
198 "dbname": self.connection.ops.quote_name(test_database_name),
199 "suffix": self.sql_table_creation_suffix(),
200 }
201 # Create the test database and connect to it.
202 with self._nodb_cursor() as cursor:
203 try:
204 self._execute_create_test_db(cursor, test_db_params, keepdb)
205 except Exception as e:
206 # if we want to keep the db, then no need to do any of the below,
207 # just return and skip it all.
208 if keepdb:
209 return test_database_name
210
211 self.log("Got an error creating the test database: %s" % e)
212 if not autoclobber:
213 confirm = input(
214 "Type 'yes' if you would like to try deleting the test "
215 "database '%s', or 'no' to cancel: " % test_database_name
216 )
217 if autoclobber or confirm == "yes":
218 try:
219 if verbosity >= 1:
220 self.log(
221 "Destroying old test database for alias %s..."
222 % (
223 self._get_database_display_str(
224 verbosity, test_database_name
225 ),
226 )
227 )
228 cursor.execute("DROP DATABASE %(dbname)s" % test_db_params)
229 self._execute_create_test_db(cursor, test_db_params, keepdb)
230 except Exception as e:
231 self.log("Got an error recreating the test database: %s" % e)
232 sys.exit(2)
233 else:
234 self.log("Tests cancelled.")
235 sys.exit(1)
236
237 return test_database_name
238
239 def clone_test_db(self, suffix, verbosity=1, autoclobber=False, keepdb=False):
240 """
241 Clone a test database.
242 """
243 source_database_name = self.connection.settings_dict["NAME"]
244
245 if verbosity >= 1:
246 action = "Cloning test database"
247 if keepdb:
248 action = "Using existing clone"
249 self.log(
250 "%s for alias %s..."
251 % (
252 action,
253 self._get_database_display_str(verbosity, source_database_name),
254 )
255 )
256
257 # We could skip this call if keepdb is True, but we instead
258 # give it the keepdb param. See create_test_db for details.
259 self._clone_test_db(suffix, verbosity, keepdb)
260
261 def get_test_db_clone_settings(self, suffix):
262 """
263 Return a modified connection settings dict for the n-th clone of a DB.
264 """
265 # When this function is called, the test database has been created
266 # already and its name has been copied to settings_dict['NAME'] so
267 # we don't need to call _get_test_db_name.
268 orig_settings_dict = self.connection.settings_dict
269 return {
270 **orig_settings_dict,
271 "NAME": "{}_{}".format(orig_settings_dict["NAME"], suffix),
272 }
273
274 def _clone_test_db(self, suffix, verbosity, keepdb=False):
275 """
276 Internal implementation - duplicate the test db tables.
277 """
278 raise NotImplementedError(
279 "The database backend doesn't support cloning databases. "
280 "Disable the option to run tests in parallel processes."
281 )
282
283 def destroy_test_db(
284 self, old_database_name=None, verbosity=1, keepdb=False, suffix=None
285 ):
286 """
287 Destroy a test database, prompting the user for confirmation if the
288 database already exists.
289 """
290 self.connection.close()
291 if suffix is None:
292 test_database_name = self.connection.settings_dict["NAME"]
293 else:
294 test_database_name = self.get_test_db_clone_settings(suffix)["NAME"]
295
296 if verbosity >= 1:
297 action = "Destroying"
298 if keepdb:
299 action = "Preserving"
300 self.log(
301 "%s test database for alias %s..."
302 % (
303 action,
304 self._get_database_display_str(verbosity, test_database_name),
305 )
306 )
307
308 # if we want to preserve the database
309 # skip the actual destroying piece.
310 if not keepdb:
311 self._destroy_test_db(test_database_name, verbosity)
312
313 # Restore the original database name
314 if old_database_name is not None:
315 settings.DATABASES[self.connection.alias]["NAME"] = old_database_name
316 self.connection.settings_dict["NAME"] = old_database_name
317
318 def _destroy_test_db(self, test_database_name, verbosity):
319 """
320 Internal implementation - remove the test db tables.
321 """
322 # Remove the test database to clean up after
323 # ourselves. Connect to the previous database (not the test database)
324 # to do so, because it's not allowed to delete a database while being
325 # connected to it.
326 with self._nodb_cursor() as cursor:
327 cursor.execute(
328 "DROP DATABASE %s" % self.connection.ops.quote_name(test_database_name)
329 )
330
331 def mark_expected_failures_and_skips(self):
332 """
333 Mark tests in Django's test suite which are expected failures on this
334 database and test which should be skipped on this database.
335 """
336 # Only load unittest if we're actually testing.
337 from unittest import expectedFailure, skip
338
339 for test_name in self.connection.features.django_test_expected_failures:
340 test_case_name, _, test_method_name = test_name.rpartition(".")
341 test_app = test_name.split(".")[0]
342 # Importing a test app that isn't installed raises RuntimeError.
343 if test_app in settings.INSTALLED_APPS:
344 test_case = import_string(test_case_name)
345 test_method = getattr(test_case, test_method_name)
346 setattr(test_case, test_method_name, expectedFailure(test_method))
347 for reason, tests in self.connection.features.django_test_skips.items():
348 for test_name in tests:
349 test_case_name, _, test_method_name = test_name.rpartition(".")
350 test_app = test_name.split(".")[0]
351 # Importing a test app that isn't installed raises RuntimeError.
352 if test_app in settings.INSTALLED_APPS:
353 test_case = import_string(test_case_name)
354 test_method = getattr(test_case, test_method_name)
355 setattr(test_case, test_method_name, skip(reason)(test_method))
356
357 def sql_table_creation_suffix(self):
358 """
359 SQL to append to the end of the test table creation statements.
360 """
361 return ""
362
363 def test_db_signature(self):
364 """
365 Return a tuple with elements of self.connection.settings_dict (a
366 DATABASES setting value) that uniquely identify a database
367 accordingly to the RDBMS particularities.
368 """
369 settings_dict = self.connection.settings_dict
370 return (
371 settings_dict["HOST"],
372 settings_dict["PORT"],
373 settings_dict["ENGINE"],
374 self._get_test_db_name(),
375 )
376
377 def setup_worker_connection(self, _worker_id):
378 settings_dict = self.get_test_db_clone_settings(str(_worker_id))
379 # connection.settings_dict must be updated in place for changes to be
380 # reflected in django.db.connections. If the following line assigned
381 # connection.settings_dict = settings_dict, new threads would connect
382 # to the default database instead of the appropriate clone.
383 self.connection.settings_dict.update(settings_dict)
384 self.connection.close()