| 1 |
import copy |
| 2 |
from decimal import Decimal |
| 3 |
|
| 4 |
from django.apps.registry import Apps |
| 5 |
from django.db import NotSupportedError |
| 6 |
from django.db.backends.base.schema import BaseDatabaseSchemaEditor |
| 7 |
from django.db.backends.ddl_references import Statement |
| 8 |
from django.db.backends.utils import strip_quotes |
| 9 |
from django.db.models import CompositePrimaryKey, UniqueConstraint |
| 10 |
|
| 11 |
|
| 12 |
class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): |
| 13 |
sql_delete_table = "DROP TABLE %(table)s" |
| 14 |
sql_create_fk = None |
| 15 |
sql_create_inline_fk = ( |
| 16 |
"REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED" |
| 17 |
) |
| 18 |
sql_create_column_inline_fk = sql_create_inline_fk |
| 19 |
sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s" |
| 20 |
sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)" |
| 21 |
sql_delete_unique = "DROP INDEX %(name)s" |
| 22 |
sql_alter_table_comment = None |
| 23 |
sql_alter_column_comment = None |
| 24 |
|
| 25 |
def __enter__(self): |
| 26 |
# Some SQLite schema alterations need foreign key constraints to be |
| 27 |
# disabled. Enforce it here for the duration of the schema edition. |
| 28 |
if not self.connection.disable_constraint_checking(): |
| 29 |
raise NotSupportedError( |
| 30 |
"SQLite schema editor cannot be used while foreign key " |
| 31 |
"constraint checks are enabled. Make sure to disable them " |
| 32 |
"before entering a transaction.atomic() context because " |
| 33 |
"SQLite does not support disabling them in the middle of " |
| 34 |
"a multi-statement transaction." |
| 35 |
) |
| 36 |
return super().__enter__() |
| 37 |
|
| 38 |
def __exit__(self, exc_type, exc_value, traceback): |
| 39 |
self.connection.check_constraints() |
| 40 |
super().__exit__(exc_type, exc_value, traceback) |
| 41 |
self.connection.enable_constraint_checking() |
| 42 |
|
| 43 |
def quote_value(self, value): |
| 44 |
# The backend "mostly works" without this function and there are use |
| 45 |
# cases for compiling Python without the sqlite3 libraries (e.g. |
| 46 |
# security hardening). |
| 47 |
try: |
| 48 |
import sqlite3 |
| 49 |
|
| 50 |
value = sqlite3.adapt(value) |
| 51 |
except ImportError: |
| 52 |
pass |
| 53 |
except sqlite3.ProgrammingError: |
| 54 |
pass |
| 55 |
# Manual emulation of SQLite parameter quoting |
| 56 |
if isinstance(value, bool): |
| 57 |
return str(int(value)) |
| 58 |
elif isinstance(value, (Decimal, float, int)): |
| 59 |
return str(value) |
| 60 |
elif isinstance(value, str): |
| 61 |
return "'%s'" % value.replace("'", "''") |
| 62 |
elif value is None: |
| 63 |
return "NULL" |
| 64 |
elif isinstance(value, (bytes, bytearray, memoryview)): |
| 65 |
# Bytes are only allowed for BLOB fields, encoded as string |
| 66 |
# literals containing hexadecimal data and preceded by a single "X" |
| 67 |
# character. |
| 68 |
return "X'%s'" % value.hex() |
| 69 |
else: |
| 70 |
raise ValueError( |
| 71 |
"Cannot quote parameter value %r of type %s" % (value, type(value)) |
| 72 |
) |
| 73 |
|
| 74 |
def prepare_default(self, value): |
| 75 |
return self.quote_value(value) |
| 76 |
|
| 77 |
def _remake_table( |
| 78 |
self, model, create_field=None, delete_field=None, alter_fields=None |
| 79 |
): |
| 80 |
""" |
| 81 |
Shortcut to transform a model from old_model into new_model |
| 82 |
|
| 83 |
This follows the correct procedure to perform non-rename or column |
| 84 |
addition operations based on SQLite's documentation |
| 85 |
|
| 86 |
https://www.sqlite.org/lang_altertable.html#caution |
| 87 |
|
| 88 |
The essential steps are: |
| 89 |
1. Create a table with the updated definition called "new__app_model" |
| 90 |
2. Copy the data from the existing "app_model" table to the new table |
| 91 |
3. Drop the "app_model" table |
| 92 |
4. Rename the "new__app_model" table to "app_model" |
| 93 |
5. Restore any index of the previous "app_model" table. |
| 94 |
""" |
| 95 |
|
| 96 |
# Self-referential fields must be recreated rather than copied from |
| 97 |
# the old model to ensure their remote_field.field_name doesn't refer |
| 98 |
# to an altered field. |
| 99 |
def is_self_referential(f): |
| 100 |
return f.is_relation and f.remote_field.model is model |
| 101 |
|
| 102 |
# Work out the new fields dict / mapping |
| 103 |
body = { |
| 104 |
f.name: f.clone() if is_self_referential(f) else f |
| 105 |
for f in model._meta.local_concrete_fields |
| 106 |
} |
| 107 |
|
| 108 |
# Since CompositePrimaryKey is not a concrete field (column is None), |
| 109 |
# it's not copied by default. |
| 110 |
pk = model._meta.pk |
| 111 |
if isinstance(pk, CompositePrimaryKey): |
| 112 |
body[pk.name] = pk.clone() |
| 113 |
|
| 114 |
# Since mapping might mix column names and default values, |
| 115 |
# its values must be already quoted. |
| 116 |
mapping = { |
| 117 |
f.column: self.quote_name(f.column) |
| 118 |
for f in model._meta.local_concrete_fields |
| 119 |
if f.generated is False |
| 120 |
} |
| 121 |
# This maps field names (not columns) for things like unique_together |
| 122 |
rename_mapping = {} |
| 123 |
# If any of the new or altered fields is introducing a new PK, |
| 124 |
# remove the old one |
| 125 |
restore_pk_field = None |
| 126 |
alter_fields = alter_fields or [] |
| 127 |
if getattr(create_field, "primary_key", False) or any( |
| 128 |
getattr(new_field, "primary_key", False) for _, new_field in alter_fields |
| 129 |
): |
| 130 |
for name, field in list(body.items()): |
| 131 |
if field.primary_key and not any( |
| 132 |
# Do not remove the old primary key when an altered field |
| 133 |
# that introduces a primary key is the same field. |
| 134 |
name == new_field.name |
| 135 |
for _, new_field in alter_fields |
| 136 |
): |
| 137 |
field.primary_key = False |
| 138 |
restore_pk_field = field |
| 139 |
if field.auto_created: |
| 140 |
del body[name] |
| 141 |
del mapping[field.column] |
| 142 |
# Add in any created fields |
| 143 |
if create_field: |
| 144 |
body[create_field.name] = create_field |
| 145 |
# Choose a default and insert it into the copy map |
| 146 |
if ( |
| 147 |
not create_field.has_db_default() |
| 148 |
and not (create_field.many_to_many or create_field.generated) |
| 149 |
and create_field.concrete |
| 150 |
): |
| 151 |
mapping[create_field.column] = self.prepare_default( |
| 152 |
self.effective_default(create_field) |
| 153 |
) |
| 154 |
# Add in any altered fields |
| 155 |
for alter_field in alter_fields: |
| 156 |
old_field, new_field = alter_field |
| 157 |
body.pop(old_field.name, None) |
| 158 |
mapping.pop(old_field.column, None) |
| 159 |
body[new_field.name] = new_field |
| 160 |
rename_mapping[old_field.name] = new_field.name |
| 161 |
if new_field.generated: |
| 162 |
continue |
| 163 |
if old_field.null and not new_field.null: |
| 164 |
if not new_field.has_db_default(): |
| 165 |
default = self.prepare_default(self.effective_default(new_field)) |
| 166 |
else: |
| 167 |
default, _ = self.db_default_sql(new_field) |
| 168 |
case_sql = "coalesce(%(col)s, %(default)s)" % { |
| 169 |
"col": self.quote_name(old_field.column), |
| 170 |
"default": default, |
| 171 |
} |
| 172 |
mapping[new_field.column] = case_sql |
| 173 |
else: |
| 174 |
mapping[new_field.column] = self.quote_name(old_field.column) |
| 175 |
# Remove any deleted fields |
| 176 |
if delete_field: |
| 177 |
del body[delete_field.name] |
| 178 |
mapping.pop(delete_field.column, None) |
| 179 |
# Remove any implicit M2M tables |
| 180 |
if ( |
| 181 |
delete_field.many_to_many |
| 182 |
and delete_field.remote_field.through._meta.auto_created |
| 183 |
): |
| 184 |
return self.delete_model(delete_field.remote_field.through) |
| 185 |
# Work inside a new app registry |
| 186 |
apps = Apps() |
| 187 |
|
| 188 |
# Work out the new value of unique_together, taking renames into |
| 189 |
# account |
| 190 |
unique_together = [ |
| 191 |
[rename_mapping.get(n, n) for n in unique] |
| 192 |
for unique in model._meta.unique_together |
| 193 |
] |
| 194 |
|
| 195 |
indexes = model._meta.indexes |
| 196 |
if delete_field: |
| 197 |
indexes = [ |
| 198 |
index for index in indexes if delete_field.name not in index.fields |
| 199 |
] |
| 200 |
|
| 201 |
constraints = list(model._meta.constraints) |
| 202 |
|
| 203 |
# Provide isolated instances of the fields to the new model body so |
| 204 |
# that the existing model's internals aren't interfered with when |
| 205 |
# the dummy model is constructed. |
| 206 |
body_copy = copy.deepcopy(body) |
| 207 |
|
| 208 |
# Construct a new model with the new fields to allow self referential |
| 209 |
# primary key to resolve to. This model won't ever be materialized as a |
| 210 |
# table and solely exists for foreign key reference resolution purposes. |
| 211 |
# This wouldn't be required if the schema editor was operating on model |
| 212 |
# states instead of rendered models. |
| 213 |
meta_contents = { |
| 214 |
"app_label": model._meta.app_label, |
| 215 |
"db_table": model._meta.db_table, |
| 216 |
"unique_together": unique_together, |
| 217 |
"indexes": indexes, |
| 218 |
"constraints": constraints, |
| 219 |
"apps": apps, |
| 220 |
} |
| 221 |
meta = type("Meta", (), meta_contents) |
| 222 |
body_copy["Meta"] = meta |
| 223 |
body_copy["__module__"] = model.__module__ |
| 224 |
type(model._meta.object_name, model.__bases__, body_copy) |
| 225 |
|
| 226 |
# Construct a model with a renamed table name. |
| 227 |
body_copy = copy.deepcopy(body) |
| 228 |
meta_contents = { |
| 229 |
"app_label": model._meta.app_label, |
| 230 |
"db_table": "new__%s" % strip_quotes(model._meta.db_table), |
| 231 |
"unique_together": unique_together, |
| 232 |
"indexes": indexes, |
| 233 |
"constraints": constraints, |
| 234 |
"apps": apps, |
| 235 |
} |
| 236 |
meta = type("Meta", (), meta_contents) |
| 237 |
body_copy["Meta"] = meta |
| 238 |
body_copy["__module__"] = model.__module__ |
| 239 |
new_model = type("New%s" % model._meta.object_name, model.__bases__, body_copy) |
| 240 |
|
| 241 |
# Remove the automatically recreated default primary key, if it has |
| 242 |
# been deleted. |
| 243 |
if delete_field and delete_field.attname == new_model._meta.pk.attname: |
| 244 |
auto_pk = new_model._meta.pk |
| 245 |
delattr(new_model, auto_pk.attname) |
| 246 |
new_model._meta.local_fields.remove(auto_pk) |
| 247 |
new_model.pk = None |
| 248 |
|
| 249 |
# Create a new table with the updated schema. |
| 250 |
self.create_model(new_model) |
| 251 |
|
| 252 |
# Copy data from the old table into the new table |
| 253 |
self.execute( |
| 254 |
"INSERT INTO %s (%s) SELECT %s FROM %s" |
| 255 |
% ( |
| 256 |
self.quote_name(new_model._meta.db_table), |
| 257 |
", ".join(self.quote_name(x) for x in mapping), |
| 258 |
", ".join(mapping.values()), |
| 259 |
self.quote_name(model._meta.db_table), |
| 260 |
) |
| 261 |
) |
| 262 |
|
| 263 |
# Delete the old table to make way for the new |
| 264 |
self.delete_model(model, handle_autom2m=False) |
| 265 |
|
| 266 |
# Rename the new table to take way for the old |
| 267 |
self.alter_db_table( |
| 268 |
new_model, |
| 269 |
new_model._meta.db_table, |
| 270 |
model._meta.db_table, |
| 271 |
) |
| 272 |
|
| 273 |
# Run deferred SQL on correct table |
| 274 |
for sql in self.deferred_sql: |
| 275 |
self.execute(sql) |
| 276 |
self.deferred_sql = [] |
| 277 |
# Fix any PK-removed field |
| 278 |
if restore_pk_field: |
| 279 |
restore_pk_field.primary_key = True |
| 280 |
|
| 281 |
def delete_model(self, model, handle_autom2m=True): |
| 282 |
if handle_autom2m: |
| 283 |
super().delete_model(model) |
| 284 |
else: |
| 285 |
# Delete the table (and only that) |
| 286 |
self.execute( |
| 287 |
self.sql_delete_table |
| 288 |
% { |
| 289 |
"table": self.quote_name(model._meta.db_table), |
| 290 |
} |
| 291 |
) |
| 292 |
# Remove all deferred statements referencing the deleted table. |
| 293 |
for sql in list(self.deferred_sql): |
| 294 |
if isinstance(sql, Statement) and sql.references_table( |
| 295 |
model._meta.db_table |
| 296 |
): |
| 297 |
self.deferred_sql.remove(sql) |
| 298 |
|
| 299 |
def add_field(self, model, field): |
| 300 |
"""Create a field on a model.""" |
| 301 |
from django.db.models.expressions import Value |
| 302 |
|
| 303 |
# Special-case implicit M2M tables. |
| 304 |
if field.many_to_many and field.remote_field.through._meta.auto_created: |
| 305 |
self.create_model(field.remote_field.through) |
| 306 |
elif isinstance(field, CompositePrimaryKey): |
| 307 |
# If a CompositePrimaryKey field was added, the existing primary key field |
| 308 |
# had to be altered too, resulting in an AddField, AlterField migration. |
| 309 |
# The table cannot be re-created on AddField, it would result in a |
| 310 |
# duplicate primary key error. |
| 311 |
return |
| 312 |
elif ( |
| 313 |
# Primary keys and unique fields are not supported in ALTER TABLE |
| 314 |
# ADD COLUMN. |
| 315 |
field.primary_key |
| 316 |
or field.unique |
| 317 |
or not field.null |
| 318 |
# Fields with default values cannot by handled by ALTER TABLE ADD |
| 319 |
# COLUMN statement because DROP DEFAULT is not supported in |
| 320 |
# ALTER TABLE. |
| 321 |
or self.effective_default(field) is not None |
| 322 |
# Fields with non-constant defaults cannot by handled by ALTER |
| 323 |
# TABLE ADD COLUMN statement. |
| 324 |
or (field.has_db_default() and not isinstance(field.db_default, Value)) |
| 325 |
): |
| 326 |
self._remake_table(model, create_field=field) |
| 327 |
else: |
| 328 |
super().add_field(model, field) |
| 329 |
|
| 330 |
def remove_field(self, model, field): |
| 331 |
""" |
| 332 |
Remove a field from a model. Usually involves deleting a column, |
| 333 |
but for M2Ms may involve deleting a table. |
| 334 |
""" |
| 335 |
# M2M fields are a special case |
| 336 |
if field.many_to_many: |
| 337 |
# For implicit M2M tables, delete the auto-created table |
| 338 |
if field.remote_field.through._meta.auto_created: |
| 339 |
self.delete_model(field.remote_field.through) |
| 340 |
# For explicit "through" M2M fields, do nothing |
| 341 |
elif ( |
| 342 |
self.connection.features.can_alter_table_drop_column |
| 343 |
# Primary keys, unique fields, indexed fields, and foreign keys are |
| 344 |
# not supported in ALTER TABLE DROP COLUMN. |
| 345 |
and not field.primary_key |
| 346 |
and not field.unique |
| 347 |
and not field.db_index |
| 348 |
and not (field.remote_field and field.db_constraint) |
| 349 |
): |
| 350 |
super().remove_field(model, field) |
| 351 |
# For everything else, remake. |
| 352 |
else: |
| 353 |
# It might not actually have a column behind it |
| 354 |
if field.db_parameters(connection=self.connection)["type"] is None: |
| 355 |
return |
| 356 |
self._remake_table(model, delete_field=field) |
| 357 |
|
| 358 |
def _alter_field( |
| 359 |
self, |
| 360 |
model, |
| 361 |
old_field, |
| 362 |
new_field, |
| 363 |
old_type, |
| 364 |
new_type, |
| 365 |
old_db_params, |
| 366 |
new_db_params, |
| 367 |
strict=False, |
| 368 |
): |
| 369 |
"""Perform a "physical" (non-ManyToMany) field update.""" |
| 370 |
# Use "ALTER TABLE ... RENAME COLUMN" if only the column name |
| 371 |
# changed and there aren't any constraints. |
| 372 |
if ( |
| 373 |
old_field.column != new_field.column |
| 374 |
and self.column_sql(model, old_field) == self.column_sql(model, new_field) |
| 375 |
and not ( |
| 376 |
old_field.remote_field |
| 377 |
and old_field.db_constraint |
| 378 |
or new_field.remote_field |
| 379 |
and new_field.db_constraint |
| 380 |
) |
| 381 |
): |
| 382 |
return self.execute( |
| 383 |
self._rename_field_sql( |
| 384 |
model._meta.db_table, old_field, new_field, new_type |
| 385 |
) |
| 386 |
) |
| 387 |
# Alter by remaking table |
| 388 |
self._remake_table(model, alter_fields=[(old_field, new_field)]) |
| 389 |
# Rebuild tables with FKs pointing to this field. |
| 390 |
old_collation = old_db_params.get("collation") |
| 391 |
new_collation = new_db_params.get("collation") |
| 392 |
if new_field.unique and ( |
| 393 |
old_type != new_type or old_collation != new_collation |
| 394 |
): |
| 395 |
related_models = set() |
| 396 |
opts = new_field.model._meta |
| 397 |
for remote_field in opts.related_objects: |
| 398 |
# Ignore self-relationship since the table was already rebuilt. |
| 399 |
if remote_field.related_model == model: |
| 400 |
continue |
| 401 |
if not remote_field.many_to_many: |
| 402 |
if remote_field.field_name == new_field.name: |
| 403 |
related_models.add(remote_field.related_model) |
| 404 |
elif new_field.primary_key and remote_field.through._meta.auto_created: |
| 405 |
related_models.add(remote_field.through) |
| 406 |
if new_field.primary_key: |
| 407 |
for many_to_many in opts.many_to_many: |
| 408 |
# Ignore self-relationship since the table was already rebuilt. |
| 409 |
if many_to_many.related_model == model: |
| 410 |
continue |
| 411 |
if many_to_many.remote_field.through._meta.auto_created: |
| 412 |
related_models.add(many_to_many.remote_field.through) |
| 413 |
for related_model in related_models: |
| 414 |
self._remake_table(related_model) |
| 415 |
|
| 416 |
def _alter_many_to_many(self, model, old_field, new_field, strict): |
| 417 |
"""Alter M2Ms to repoint their to= endpoints.""" |
| 418 |
if ( |
| 419 |
old_field.remote_field.through._meta.db_table |
| 420 |
== new_field.remote_field.through._meta.db_table |
| 421 |
): |
| 422 |
# The field name didn't change, but some options did, so we have to |
| 423 |
# propagate this altering. |
| 424 |
self._remake_table( |
| 425 |
old_field.remote_field.through, |
| 426 |
alter_fields=[ |
| 427 |
( |
| 428 |
# The field that points to the target model is needed, |
| 429 |
# so that table can be remade with the new m2m field - |
| 430 |
# this is m2m_reverse_field_name(). |
| 431 |
old_field.remote_field.through._meta.get_field( |
| 432 |
old_field.m2m_reverse_field_name() |
| 433 |
), |
| 434 |
new_field.remote_field.through._meta.get_field( |
| 435 |
new_field.m2m_reverse_field_name() |
| 436 |
), |
| 437 |
), |
| 438 |
( |
| 439 |
# The field that points to the model itself is needed, |
| 440 |
# so that table can be remade with the new self field - |
| 441 |
# this is m2m_field_name(). |
| 442 |
old_field.remote_field.through._meta.get_field( |
| 443 |
old_field.m2m_field_name() |
| 444 |
), |
| 445 |
new_field.remote_field.through._meta.get_field( |
| 446 |
new_field.m2m_field_name() |
| 447 |
), |
| 448 |
), |
| 449 |
], |
| 450 |
) |
| 451 |
return |
| 452 |
|
| 453 |
# Make a new through table |
| 454 |
self.create_model(new_field.remote_field.through) |
| 455 |
# Copy the data across |
| 456 |
self.execute( |
| 457 |
"INSERT INTO %s (%s) SELECT %s FROM %s" |
| 458 |
% ( |
| 459 |
self.quote_name(new_field.remote_field.through._meta.db_table), |
| 460 |
", ".join( |
| 461 |
[ |
| 462 |
"id", |
| 463 |
new_field.m2m_column_name(), |
| 464 |
new_field.m2m_reverse_name(), |
| 465 |
] |
| 466 |
), |
| 467 |
", ".join( |
| 468 |
[ |
| 469 |
"id", |
| 470 |
old_field.m2m_column_name(), |
| 471 |
old_field.m2m_reverse_name(), |
| 472 |
] |
| 473 |
), |
| 474 |
self.quote_name(old_field.remote_field.through._meta.db_table), |
| 475 |
) |
| 476 |
) |
| 477 |
# Delete the old through table |
| 478 |
self.delete_model(old_field.remote_field.through) |
| 479 |
|
| 480 |
def add_constraint(self, model, constraint): |
| 481 |
if isinstance(constraint, UniqueConstraint) and ( |
| 482 |
constraint.condition |
| 483 |
or constraint.contains_expressions |
| 484 |
or constraint.include |
| 485 |
or constraint.deferrable |
| 486 |
): |
| 487 |
super().add_constraint(model, constraint) |
| 488 |
else: |
| 489 |
self._remake_table(model) |
| 490 |
|
| 491 |
def remove_constraint(self, model, constraint): |
| 492 |
if isinstance(constraint, UniqueConstraint) and ( |
| 493 |
constraint.condition |
| 494 |
or constraint.contains_expressions |
| 495 |
or constraint.include |
| 496 |
or constraint.deferrable |
| 497 |
): |
| 498 |
super().remove_constraint(model, constraint) |
| 499 |
else: |
| 500 |
self._remake_table(model) |
| 501 |
|
| 502 |
def _collate_sql(self, collation): |
| 503 |
return "COLLATE " + collation |