| 1 |
from django.apps.registry import apps as global_apps |
| 2 |
from django.db import migrations, router |
| 3 |
|
| 4 |
from .exceptions import InvalidMigrationPlan |
| 5 |
from .loader import MigrationLoader |
| 6 |
from .recorder import MigrationRecorder |
| 7 |
from .state import ProjectState |
| 8 |
|
| 9 |
|
| 10 |
class MigrationExecutor: |
| 11 |
""" |
| 12 |
End-to-end migration execution - load migrations and run them up or down |
| 13 |
to a specified set of targets. |
| 14 |
""" |
| 15 |
|
| 16 |
def __init__(self, connection, progress_callback=None): |
| 17 |
self.connection = connection |
| 18 |
self.loader = MigrationLoader(self.connection) |
| 19 |
self.recorder = MigrationRecorder(self.connection) |
| 20 |
self.progress_callback = progress_callback |
| 21 |
|
| 22 |
def migration_plan(self, targets, clean_start=False): |
| 23 |
""" |
| 24 |
Given a set of targets, return a list of (Migration instance, backwards?). |
| 25 |
""" |
| 26 |
plan = [] |
| 27 |
if clean_start: |
| 28 |
applied = {} |
| 29 |
else: |
| 30 |
applied = dict(self.loader.applied_migrations) |
| 31 |
for target in targets: |
| 32 |
# If the target is (app_label, None), that means unmigrate everything |
| 33 |
if target[1] is None: |
| 34 |
for root in self.loader.graph.root_nodes(): |
| 35 |
if root[0] == target[0]: |
| 36 |
for migration in self.loader.graph.backwards_plan(root): |
| 37 |
if migration in applied: |
| 38 |
plan.append((self.loader.graph.nodes[migration], True)) |
| 39 |
applied.pop(migration) |
| 40 |
# If the migration is already applied, do backwards mode, |
| 41 |
# otherwise do forwards mode. |
| 42 |
elif target in applied: |
| 43 |
# If the target is missing, it's likely a replaced migration. |
| 44 |
# Reload the graph without replacements. |
| 45 |
if ( |
| 46 |
self.loader.replace_migrations |
| 47 |
and target not in self.loader.graph.node_map |
| 48 |
): |
| 49 |
self.loader.replace_migrations = False |
| 50 |
self.loader.build_graph() |
| 51 |
return self.migration_plan(targets, clean_start=clean_start) |
| 52 |
# Don't migrate backwards all the way to the target node (that |
| 53 |
# may roll back dependencies in other apps that don't need to |
| 54 |
# be rolled back); instead roll back through target's immediate |
| 55 |
# child(ren) in the same app, and no further. |
| 56 |
next_in_app = sorted( |
| 57 |
n |
| 58 |
for n in self.loader.graph.node_map[target].children |
| 59 |
if n[0] == target[0] |
| 60 |
) |
| 61 |
for node in next_in_app: |
| 62 |
for migration in self.loader.graph.backwards_plan(node): |
| 63 |
if migration in applied: |
| 64 |
plan.append((self.loader.graph.nodes[migration], True)) |
| 65 |
applied.pop(migration) |
| 66 |
else: |
| 67 |
for migration in self.loader.graph.forwards_plan(target): |
| 68 |
if migration not in applied: |
| 69 |
plan.append((self.loader.graph.nodes[migration], False)) |
| 70 |
applied[migration] = self.loader.graph.nodes[migration] |
| 71 |
return plan |
| 72 |
|
| 73 |
def _create_project_state(self, with_applied_migrations=False): |
| 74 |
""" |
| 75 |
Create a project state including all the applications without |
| 76 |
migrations and applied migrations if with_applied_migrations=True. |
| 77 |
""" |
| 78 |
state = ProjectState(real_apps=self.loader.unmigrated_apps) |
| 79 |
if with_applied_migrations: |
| 80 |
# Create the forwards plan Django would follow on an empty database |
| 81 |
full_plan = self.migration_plan( |
| 82 |
self.loader.graph.leaf_nodes(), clean_start=True |
| 83 |
) |
| 84 |
applied_migrations = { |
| 85 |
self.loader.graph.nodes[key] |
| 86 |
for key in self.loader.applied_migrations |
| 87 |
if key in self.loader.graph.nodes |
| 88 |
} |
| 89 |
for migration, _ in full_plan: |
| 90 |
if migration in applied_migrations: |
| 91 |
migration.mutate_state(state, preserve=False) |
| 92 |
return state |
| 93 |
|
| 94 |
def migrate(self, targets, plan=None, state=None, fake=False, fake_initial=False): |
| 95 |
""" |
| 96 |
Migrate the database up to the given targets. |
| 97 |
|
| 98 |
Django first needs to create all project states before a migration is |
| 99 |
(un)applied and in a second step run all the database operations. |
| 100 |
""" |
| 101 |
# The django_migrations table must be present to record applied |
| 102 |
# migrations, but don't create it if there are no migrations to apply. |
| 103 |
if plan == []: |
| 104 |
if not self.recorder.has_table(): |
| 105 |
return self._create_project_state(with_applied_migrations=False) |
| 106 |
else: |
| 107 |
self.recorder.ensure_schema() |
| 108 |
|
| 109 |
if plan is None: |
| 110 |
plan = self.migration_plan(targets) |
| 111 |
# Create the forwards plan Django would follow on an empty database |
| 112 |
full_plan = self.migration_plan( |
| 113 |
self.loader.graph.leaf_nodes(), clean_start=True |
| 114 |
) |
| 115 |
|
| 116 |
all_forwards = all(not backwards for mig, backwards in plan) |
| 117 |
all_backwards = all(backwards for mig, backwards in plan) |
| 118 |
|
| 119 |
if not plan: |
| 120 |
if state is None: |
| 121 |
# The resulting state should include applied migrations. |
| 122 |
state = self._create_project_state(with_applied_migrations=True) |
| 123 |
elif all_forwards == all_backwards: |
| 124 |
# This should only happen if there's a mixed plan |
| 125 |
raise InvalidMigrationPlan( |
| 126 |
"Migration plans with both forwards and backwards migrations " |
| 127 |
"are not supported. Please split your migration process into " |
| 128 |
"separate plans of only forwards OR backwards migrations.", |
| 129 |
plan, |
| 130 |
) |
| 131 |
elif all_forwards: |
| 132 |
if state is None: |
| 133 |
# The resulting state should still include applied migrations. |
| 134 |
state = self._create_project_state(with_applied_migrations=True) |
| 135 |
state = self._migrate_all_forwards( |
| 136 |
state, plan, full_plan, fake=fake, fake_initial=fake_initial |
| 137 |
) |
| 138 |
else: |
| 139 |
# No need to check for `elif all_backwards` here, as that condition |
| 140 |
# would always evaluate to true. |
| 141 |
state = self._migrate_all_backwards(plan, full_plan, fake=fake) |
| 142 |
|
| 143 |
self.check_replacements() |
| 144 |
|
| 145 |
return state |
| 146 |
|
| 147 |
def _migrate_all_forwards(self, state, plan, full_plan, fake, fake_initial): |
| 148 |
""" |
| 149 |
Take a list of 2-tuples of the form (migration instance, False) and |
| 150 |
apply them in the order they occur in the full_plan. |
| 151 |
""" |
| 152 |
migrations_to_run = {m[0] for m in plan} |
| 153 |
for migration, _ in full_plan: |
| 154 |
if not migrations_to_run: |
| 155 |
# We remove every migration that we applied from these sets so |
| 156 |
# that we can bail out once the last migration has been applied |
| 157 |
# and don't always run until the very end of the migration |
| 158 |
# process. |
| 159 |
break |
| 160 |
if migration in migrations_to_run: |
| 161 |
if "apps" not in state.__dict__: |
| 162 |
if self.progress_callback: |
| 163 |
self.progress_callback("render_start") |
| 164 |
state.apps # Render all -- performance critical |
| 165 |
if self.progress_callback: |
| 166 |
self.progress_callback("render_success") |
| 167 |
state = self.apply_migration( |
| 168 |
state, migration, fake=fake, fake_initial=fake_initial |
| 169 |
) |
| 170 |
migrations_to_run.remove(migration) |
| 171 |
|
| 172 |
return state |
| 173 |
|
| 174 |
def _migrate_all_backwards(self, plan, full_plan, fake): |
| 175 |
""" |
| 176 |
Take a list of 2-tuples of the form (migration instance, True) and |
| 177 |
unapply them in reverse order they occur in the full_plan. |
| 178 |
|
| 179 |
Since unapplying a migration requires the project state prior to that |
| 180 |
migration, Django will compute the migration states before each of them |
| 181 |
in a first run over the plan and then unapply them in a second run over |
| 182 |
the plan. |
| 183 |
""" |
| 184 |
migrations_to_run = {m[0] for m in plan} |
| 185 |
# Holds all migration states prior to the migrations being unapplied |
| 186 |
states = {} |
| 187 |
state = self._create_project_state() |
| 188 |
applied_migrations = { |
| 189 |
self.loader.graph.nodes[key] |
| 190 |
for key in self.loader.applied_migrations |
| 191 |
if key in self.loader.graph.nodes |
| 192 |
} |
| 193 |
if self.progress_callback: |
| 194 |
self.progress_callback("render_start") |
| 195 |
for migration, _ in full_plan: |
| 196 |
if not migrations_to_run: |
| 197 |
# We remove every migration that we applied from this set so |
| 198 |
# that we can bail out once the last migration has been applied |
| 199 |
# and don't always run until the very end of the migration |
| 200 |
# process. |
| 201 |
break |
| 202 |
if migration in migrations_to_run: |
| 203 |
if "apps" not in state.__dict__: |
| 204 |
state.apps # Render all -- performance critical |
| 205 |
# The state before this migration |
| 206 |
states[migration] = state |
| 207 |
# The old state keeps as-is, we continue with the new state |
| 208 |
state = migration.mutate_state(state, preserve=True) |
| 209 |
migrations_to_run.remove(migration) |
| 210 |
elif migration in applied_migrations: |
| 211 |
# Only mutate the state if the migration is actually applied |
| 212 |
# to make sure the resulting state doesn't include changes |
| 213 |
# from unrelated migrations. |
| 214 |
migration.mutate_state(state, preserve=False) |
| 215 |
if self.progress_callback: |
| 216 |
self.progress_callback("render_success") |
| 217 |
|
| 218 |
for migration, _ in plan: |
| 219 |
self.unapply_migration(states[migration], migration, fake=fake) |
| 220 |
applied_migrations.remove(migration) |
| 221 |
|
| 222 |
# Generate the post migration state by starting from the state before |
| 223 |
# the last migration is unapplied and mutating it to include all the |
| 224 |
# remaining applied migrations. |
| 225 |
last_unapplied_migration = plan[-1][0] |
| 226 |
state = states[last_unapplied_migration] |
| 227 |
# Avoid mutating state with apps rendered as it's an expensive |
| 228 |
# operation. |
| 229 |
del state.apps |
| 230 |
for index, (migration, _) in enumerate(full_plan): |
| 231 |
if migration == last_unapplied_migration: |
| 232 |
for migration, _ in full_plan[index:]: |
| 233 |
if migration in applied_migrations: |
| 234 |
migration.mutate_state(state, preserve=False) |
| 235 |
break |
| 236 |
|
| 237 |
return state |
| 238 |
|
| 239 |
def apply_migration(self, state, migration, fake=False, fake_initial=False): |
| 240 |
"""Run a migration forwards.""" |
| 241 |
migration_recorded = False |
| 242 |
if self.progress_callback: |
| 243 |
self.progress_callback("apply_start", migration, fake) |
| 244 |
if not fake: |
| 245 |
if fake_initial: |
| 246 |
# Test to see if this is an already-applied initial migration |
| 247 |
applied, state = self.detect_soft_applied(state, migration) |
| 248 |
if applied: |
| 249 |
fake = True |
| 250 |
if not fake: |
| 251 |
# Alright, do it normally |
| 252 |
with self.connection.schema_editor( |
| 253 |
atomic=migration.atomic |
| 254 |
) as schema_editor: |
| 255 |
state = migration.apply(state, schema_editor) |
| 256 |
if not schema_editor.deferred_sql: |
| 257 |
self.record_migration(migration) |
| 258 |
migration_recorded = True |
| 259 |
if not migration_recorded: |
| 260 |
self.record_migration(migration) |
| 261 |
# Report progress |
| 262 |
if self.progress_callback: |
| 263 |
self.progress_callback("apply_success", migration, fake) |
| 264 |
return state |
| 265 |
|
| 266 |
def record_migration(self, migration): |
| 267 |
# For replacement migrations, record individual statuses |
| 268 |
if migration.replaces: |
| 269 |
for app_label, name in migration.replaces: |
| 270 |
self.recorder.record_applied(app_label, name) |
| 271 |
else: |
| 272 |
self.recorder.record_applied(migration.app_label, migration.name) |
| 273 |
|
| 274 |
def unapply_migration(self, state, migration, fake=False): |
| 275 |
"""Run a migration backwards.""" |
| 276 |
if self.progress_callback: |
| 277 |
self.progress_callback("unapply_start", migration, fake) |
| 278 |
if not fake: |
| 279 |
with self.connection.schema_editor( |
| 280 |
atomic=migration.atomic |
| 281 |
) as schema_editor: |
| 282 |
state = migration.unapply(state, schema_editor) |
| 283 |
# For replacement migrations, also record individual statuses. |
| 284 |
if migration.replaces: |
| 285 |
for app_label, name in migration.replaces: |
| 286 |
self.recorder.record_unapplied(app_label, name) |
| 287 |
self.recorder.record_unapplied(migration.app_label, migration.name) |
| 288 |
# Report progress |
| 289 |
if self.progress_callback: |
| 290 |
self.progress_callback("unapply_success", migration, fake) |
| 291 |
return state |
| 292 |
|
| 293 |
def check_replacements(self): |
| 294 |
""" |
| 295 |
Mark replacement migrations applied if their replaced set all are. |
| 296 |
|
| 297 |
Do this unconditionally on every migrate, rather than just when |
| 298 |
migrations are applied or unapplied, to correctly handle the case |
| 299 |
when a new squash migration is pushed to a deployment that already had |
| 300 |
all its replaced migrations applied. In this case no new migration will |
| 301 |
be applied, but the applied state of the squashed migration must be |
| 302 |
maintained. |
| 303 |
""" |
| 304 |
applied = self.recorder.applied_migrations() |
| 305 |
for key, migration in self.loader.replacements.items(): |
| 306 |
all_applied = all(m in applied for m in migration.replaces) |
| 307 |
if all_applied and key not in applied: |
| 308 |
self.recorder.record_applied(*key) |
| 309 |
|
| 310 |
def detect_soft_applied(self, project_state, migration): |
| 311 |
""" |
| 312 |
Test whether a migration has been implicitly applied - that the |
| 313 |
tables or columns it would create exist. This is intended only for use |
| 314 |
on initial migrations (as it only looks for CreateModel and AddField). |
| 315 |
""" |
| 316 |
|
| 317 |
def should_skip_detecting_model(migration, model): |
| 318 |
""" |
| 319 |
No need to detect tables for proxy models, unmanaged models, or |
| 320 |
models that can't be migrated on the current database. |
| 321 |
""" |
| 322 |
return ( |
| 323 |
model._meta.proxy |
| 324 |
or not model._meta.managed |
| 325 |
or not router.allow_migrate( |
| 326 |
self.connection.alias, |
| 327 |
migration.app_label, |
| 328 |
model_name=model._meta.model_name, |
| 329 |
) |
| 330 |
) |
| 331 |
|
| 332 |
if migration.initial is None: |
| 333 |
# Bail if the migration isn't the first one in its app |
| 334 |
if any(app == migration.app_label for app, name in migration.dependencies): |
| 335 |
return False, project_state |
| 336 |
elif migration.initial is False: |
| 337 |
# Bail if it's NOT an initial migration |
| 338 |
return False, project_state |
| 339 |
|
| 340 |
if project_state is None: |
| 341 |
after_state = self.loader.project_state( |
| 342 |
(migration.app_label, migration.name), at_end=True |
| 343 |
) |
| 344 |
else: |
| 345 |
after_state = migration.mutate_state(project_state) |
| 346 |
apps = after_state.apps |
| 347 |
found_create_model_migration = False |
| 348 |
found_add_field_migration = False |
| 349 |
fold_identifier_case = self.connection.features.ignores_table_name_case |
| 350 |
with self.connection.cursor() as cursor: |
| 351 |
existing_table_names = set( |
| 352 |
self.connection.introspection.table_names(cursor) |
| 353 |
) |
| 354 |
if fold_identifier_case: |
| 355 |
existing_table_names = { |
| 356 |
name.casefold() for name in existing_table_names |
| 357 |
} |
| 358 |
# Make sure all create model and add field operations are done |
| 359 |
for operation in migration.operations: |
| 360 |
if isinstance(operation, migrations.CreateModel): |
| 361 |
model = apps.get_model(migration.app_label, operation.name) |
| 362 |
if model._meta.swapped: |
| 363 |
# We have to fetch the model to test with from the |
| 364 |
# main app cache, as it's not a direct dependency. |
| 365 |
model = global_apps.get_model(model._meta.swapped) |
| 366 |
if should_skip_detecting_model(migration, model): |
| 367 |
continue |
| 368 |
db_table = model._meta.db_table |
| 369 |
if fold_identifier_case: |
| 370 |
db_table = db_table.casefold() |
| 371 |
if db_table not in existing_table_names: |
| 372 |
return False, project_state |
| 373 |
found_create_model_migration = True |
| 374 |
elif isinstance(operation, migrations.AddField): |
| 375 |
model = apps.get_model(migration.app_label, operation.model_name) |
| 376 |
if model._meta.swapped: |
| 377 |
# We have to fetch the model to test with from the |
| 378 |
# main app cache, as it's not a direct dependency. |
| 379 |
model = global_apps.get_model(model._meta.swapped) |
| 380 |
if should_skip_detecting_model(migration, model): |
| 381 |
continue |
| 382 |
|
| 383 |
table = model._meta.db_table |
| 384 |
field = model._meta.get_field(operation.name) |
| 385 |
|
| 386 |
# Handle implicit many-to-many tables created by AddField. |
| 387 |
if field.many_to_many: |
| 388 |
through_db_table = field.remote_field.through._meta.db_table |
| 389 |
if fold_identifier_case: |
| 390 |
through_db_table = through_db_table.casefold() |
| 391 |
if through_db_table not in existing_table_names: |
| 392 |
return False, project_state |
| 393 |
else: |
| 394 |
found_add_field_migration = True |
| 395 |
continue |
| 396 |
with self.connection.cursor() as cursor: |
| 397 |
columns = self.connection.introspection.get_table_description( |
| 398 |
cursor, table |
| 399 |
) |
| 400 |
for column in columns: |
| 401 |
field_column = field.column |
| 402 |
column_name = column.name |
| 403 |
if fold_identifier_case: |
| 404 |
column_name = column_name.casefold() |
| 405 |
field_column = field_column.casefold() |
| 406 |
if column_name == field_column: |
| 407 |
found_add_field_migration = True |
| 408 |
break |
| 409 |
else: |
| 410 |
return False, project_state |
| 411 |
# If we get this far and we found at least one CreateModel or AddField |
| 412 |
# migration, the migration is considered implicitly applied. |
| 413 |
return (found_create_model_migration or found_add_field_migration), after_state |