Debian Codegraph @

django/db/migrations/loader.py

raw
1 import pkgutil
2 import sys
3 from importlib import import_module, reload
4
5 from django.apps import apps
6 from django.conf import settings
7 from django.db.migrations.graph import MigrationGraph
8 from django.db.migrations.recorder import MigrationRecorder
9
10 from .exceptions import (
11 AmbiguityError,
12 BadMigrationError,
13 InconsistentMigrationHistory,
14 NodeNotFoundError,
15 )
16
17 MIGRATIONS_MODULE_NAME = "migrations"
18
19
20 class MigrationLoader:
21 """
22 Load migration files from disk and their status from the database.
23
24 Migration files are expected to live in the "migrations" directory of
25 an app. Their names are entirely unimportant from a code perspective,
26 but will probably follow the 1234_name.py convention.
27
28 On initialization, this class will scan those directories, and open and
29 read the Python files, looking for a class called Migration, which should
30 inherit from django.db.migrations.Migration. See
31 django.db.migrations.migration for what that looks like.
32
33 Some migrations will be marked as "replacing" another set of migrations.
34 These are loaded into a separate set of migrations away from the main ones.
35 If all the migrations they replace are either unapplied or missing from
36 disk, then they are injected into the main set, replacing the named migrations.
37 Any dependency pointers to the replaced migrations are re-pointed to the
38 new migration.
39
40 This does mean that this class MUST also talk to the database as well as
41 to disk, but this is probably fine. We're already not just operating
42 in memory.
43 """
44
45 def __init__(
46 self,
47 connection,
48 load=True,
49 ignore_no_migrations=False,
50 replace_migrations=True,
51 ):
52 self.connection = connection
53 self.disk_migrations = None
54 self.applied_migrations = None
55 self.ignore_no_migrations = ignore_no_migrations
56 self.replace_migrations = replace_migrations
57 if load:
58 self.build_graph()
59
60 @classmethod
61 def migrations_module(cls, app_label):
62 """
63 Return the path to the migrations module for the specified app_label
64 and a boolean indicating if the module is specified in
65 settings.MIGRATION_MODULE.
66 """
67 if app_label in settings.MIGRATION_MODULES:
68 return settings.MIGRATION_MODULES[app_label], True
69 else:
70 app_package_name = apps.get_app_config(app_label).name
71 return "%s.%s" % (app_package_name, MIGRATIONS_MODULE_NAME), False
72
73 def load_disk(self):
74 """Load the migrations from all INSTALLED_APPS from disk."""
75 self.disk_migrations = {}
76 self.unmigrated_apps = set()
77 self.migrated_apps = set()
78 for app_config in apps.get_app_configs():
79 # Get the migrations module directory
80 module_name, explicit = self.migrations_module(app_config.label)
81 if module_name is None:
82 self.unmigrated_apps.add(app_config.label)
83 continue
84 was_loaded = module_name in sys.modules
85 try:
86 module = import_module(module_name)
87 except ModuleNotFoundError as e:
88 if (explicit and self.ignore_no_migrations) or (
89 not explicit and MIGRATIONS_MODULE_NAME in e.name.split(".")
90 ):
91 self.unmigrated_apps.add(app_config.label)
92 continue
93 raise
94 else:
95 # Module is not a package (e.g. migrations.py).
96 if not hasattr(module, "__path__"):
97 self.unmigrated_apps.add(app_config.label)
98 continue
99 # Empty directories are namespaces. Namespace packages have no
100 # __file__ and don't use a list for __path__. See
101 # https://docs.python.org/3/reference/import.html#namespace-packages
102 if getattr(module, "__file__", None) is None and not isinstance(
103 module.__path__, list
104 ):
105 self.unmigrated_apps.add(app_config.label)
106 continue
107 # Force a reload if it's already loaded (tests need this)
108 if was_loaded:
109 reload(module)
110 self.migrated_apps.add(app_config.label)
111 migration_names = {
112 name
113 for _, name, is_pkg in pkgutil.iter_modules(module.__path__)
114 if not is_pkg and name[0] not in "_~"
115 }
116 # Load migrations
117 for migration_name in migration_names:
118 migration_path = "%s.%s" % (module_name, migration_name)
119 try:
120 migration_module = import_module(migration_path)
121 except ImportError as e:
122 if "bad magic number" in str(e):
123 raise ImportError(
124 "Couldn't import %r as it appears to be a stale "
125 ".pyc file." % migration_path
126 ) from e
127 else:
128 raise
129 if not hasattr(migration_module, "Migration"):
130 raise BadMigrationError(
131 "Migration %s in app %s has no Migration class"
132 % (migration_name, app_config.label)
133 )
134 self.disk_migrations[app_config.label, migration_name] = (
135 migration_module.Migration(
136 migration_name,
137 app_config.label,
138 )
139 )
140
141 def get_migration(self, app_label, name_prefix):
142 """Return the named migration or raise NodeNotFoundError."""
143 return self.graph.nodes[app_label, name_prefix]
144
145 def get_migration_by_prefix(self, app_label, name_prefix):
146 """
147 Return the migration(s) which match the given app label and name_prefix.
148 """
149 # Do the search
150 results = []
151 for migration_app_label, migration_name in self.disk_migrations:
152 if migration_app_label == app_label and migration_name.startswith(
153 name_prefix
154 ):
155 results.append((migration_app_label, migration_name))
156 if len(results) > 1:
157 raise AmbiguityError(
158 "There is more than one migration for '%s' with the prefix '%s'"
159 % (app_label, name_prefix)
160 )
161 elif not results:
162 raise KeyError(
163 f"There is no migration for '{app_label}' with the prefix "
164 f"'{name_prefix}'"
165 )
166 else:
167 return self.disk_migrations[results[0]]
168
169 def check_key(self, key, current_app):
170 if (key[1] != "__first__" and key[1] != "__latest__") or key in self.graph:
171 return key
172 # Special-case __first__, which means "the first migration" for
173 # migrated apps, and is ignored for unmigrated apps. It allows
174 # makemigrations to declare dependencies on apps before they even have
175 # migrations.
176 if key[0] == current_app:
177 # Ignore __first__ references to the same app (#22325)
178 return
179 if key[0] in self.unmigrated_apps:
180 # This app isn't migrated, but something depends on it.
181 # The models will get auto-added into the state, though
182 # so we're fine.
183 return
184 if key[0] in self.migrated_apps:
185 try:
186 if key[1] == "__first__":
187 return self.graph.root_nodes(key[0])[0]
188 else: # "__latest__"
189 return self.graph.leaf_nodes(key[0])[0]
190 except IndexError:
191 if self.ignore_no_migrations:
192 return None
193 else:
194 raise ValueError(
195 "Dependency on app with no migrations: %s" % key[0]
196 )
197 raise ValueError("Dependency on unknown app: %s" % key[0])
198
199 def add_internal_dependencies(self, key, migration):
200 """
201 Internal dependencies need to be added first to ensure `__first__`
202 dependencies find the correct root node.
203 """
204 for parent in migration.dependencies:
205 # Ignore __first__ references to the same app.
206 if parent[0] == key[0] and parent[1] != "__first__":
207 self.graph.add_dependency(migration, key, parent, skip_validation=True)
208
209 def add_external_dependencies(self, key, migration):
210 for parent in migration.dependencies:
211 # Skip internal dependencies
212 if key[0] == parent[0]:
213 continue
214 parent = self.check_key(parent, key[0])
215 if parent is not None:
216 self.graph.add_dependency(migration, key, parent, skip_validation=True)
217 for child in migration.run_before:
218 child = self.check_key(child, key[0])
219 if child is not None:
220 self.graph.add_dependency(migration, child, key, skip_validation=True)
221
222 def build_graph(self):
223 """
224 Build a migration dependency graph using both the disk and database.
225 You'll need to rebuild the graph if you apply migrations. This isn't
226 usually a problem as generally migration stuff runs in a one-shot process.
227 """
228 # Load disk data
229 self.load_disk()
230 # Load database data
231 if self.connection is None:
232 self.applied_migrations = {}
233 else:
234 recorder = MigrationRecorder(self.connection)
235 self.applied_migrations = recorder.applied_migrations()
236 # To start, populate the migration graph with nodes for ALL migrations
237 # and their dependencies. Also make note of replacing migrations at this step.
238 self.graph = MigrationGraph()
239 self.replacements = {}
240 for key, migration in self.disk_migrations.items():
241 self.graph.add_node(key, migration)
242 # Replacing migrations.
243 if migration.replaces:
244 self.replacements[key] = migration
245 for key, migration in self.disk_migrations.items():
246 # Internal (same app) dependencies.
247 self.add_internal_dependencies(key, migration)
248 # Add external dependencies now that the internal ones have been resolved.
249 for key, migration in self.disk_migrations.items():
250 self.add_external_dependencies(key, migration)
251 # Carry out replacements where possible and if enabled.
252 if self.replace_migrations:
253 for key, migration in self.replacements.items():
254 # Get applied status of each of this migration's replacement
255 # targets.
256 applied_statuses = [
257 (target in self.applied_migrations) for target in migration.replaces
258 ]
259 # The replacing migration is only marked as applied if all of
260 # its replacement targets are.
261 if all(applied_statuses):
262 self.applied_migrations[key] = migration
263 else:
264 self.applied_migrations.pop(key, None)
265 # A replacing migration can be used if either all or none of
266 # its replacement targets have been applied.
267 if all(applied_statuses) or (not any(applied_statuses)):
268 self.graph.remove_replaced_nodes(key, migration.replaces)
269 else:
270 # This replacing migration cannot be used because it is
271 # partially applied. Remove it from the graph and remap
272 # dependencies to it (#25945).
273 self.graph.remove_replacement_node(key, migration.replaces)
274 # Ensure the graph is consistent.
275 try:
276 self.graph.validate_consistency()
277 except NodeNotFoundError as exc:
278 # Check if the missing node could have been replaced by any squash
279 # migration but wasn't because the squash migration was partially
280 # applied before. In that case raise a more understandable exception
281 # (#23556).
282 # Get reverse replacements.
283 reverse_replacements = {}
284 for key, migration in self.replacements.items():
285 for replaced in migration.replaces:
286 reverse_replacements.setdefault(replaced, set()).add(key)
287 # Try to reraise exception with more detail.
288 if exc.node in reverse_replacements:
289 candidates = reverse_replacements.get(exc.node, set())
290 is_replaced = any(
291 candidate in self.graph.nodes for candidate in candidates
292 )
293 if not is_replaced:
294 tries = ", ".join("%s.%s" % c for c in candidates)
295 raise NodeNotFoundError(
296 "Migration {0} depends on nonexistent node ('{1}', '{2}'). "
297 "Django tried to replace migration {1}.{2} with any of [{3}] "
298 "but wasn't able to because some of the replaced migrations "
299 "are already applied.".format(
300 exc.origin, exc.node[0], exc.node[1], tries
301 ),
302 exc.node,
303 ) from exc
304 raise
305 self.graph.ensure_not_cyclic()
306
307 def check_consistent_history(self, connection):
308 """
309 Raise InconsistentMigrationHistory if any applied migrations have
310 unapplied dependencies.
311 """
312 recorder = MigrationRecorder(connection)
313 applied = recorder.applied_migrations()
314 for migration in applied:
315 # If the migration is unknown, skip it.
316 if migration not in self.graph.nodes:
317 continue
318 for parent in self.graph.node_map[migration].parents:
319 if parent not in applied:
320 # Skip unapplied squashed migrations that have all of their
321 # `replaces` applied.
322 if parent in self.replacements:
323 if all(
324 m in applied for m in self.replacements[parent].replaces
325 ):
326 continue
327 raise InconsistentMigrationHistory(
328 "Migration {}.{} is applied before its dependency "
329 "{}.{} on database '{}'.".format(
330 migration[0],
331 migration[1],
332 parent[0],
333 parent[1],
334 connection.alias,
335 )
336 )
337
338 def detect_conflicts(self):
339 """
340 Look through the loaded graph and detect any conflicts - apps
341 with more than one leaf migration. Return a dict of the app labels
342 that conflict with the migration names that conflict.
343 """
344 seen_apps = {}
345 conflicting_apps = set()
346 for app_label, migration_name in self.graph.leaf_nodes():
347 if app_label in seen_apps:
348 conflicting_apps.add(app_label)
349 seen_apps.setdefault(app_label, set()).add(migration_name)
350 return {
351 app_label: sorted(seen_apps[app_label]) for app_label in conflicting_apps
352 }
353
354 def project_state(self, nodes=None, at_end=True):
355 """
356 Return a ProjectState object representing the most recent state
357 that the loaded migrations represent.
358
359 See graph.make_state() for the meaning of "nodes" and "at_end".
360 """
361 return self.graph.make_state(
362 nodes=nodes, at_end=at_end, real_apps=self.unmigrated_apps
363 )
364
365 def collect_sql(self, plan):
366 """
367 Take a migration plan and return a list of collected SQL statements
368 that represent the best-efforts version of that plan.
369 """
370 statements = []
371 state = None
372 for migration, backwards in plan:
373 with self.connection.schema_editor(
374 collect_sql=True, atomic=migration.atomic
375 ) as schema_editor:
376 if state is None:
377 state = self.project_state(
378 (migration.app_label, migration.name), at_end=False
379 )
380 if not backwards:
381 state = migration.apply(state, schema_editor, collect_sql=True)
382 else:
383 state = migration.unapply(state, schema_editor, collect_sql=True)
384 statements.extend(schema_editor.collected_sql)
385 return statements