| 1 |
from collections import Counter, defaultdict |
| 2 |
from functools import partial, reduce |
| 3 |
from itertools import chain |
| 4 |
from operator import attrgetter, or_ |
| 5 |
|
| 6 |
from django.db import IntegrityError, connections, models, transaction |
| 7 |
from django.db.models import query_utils, signals, sql |
| 8 |
|
| 9 |
|
| 10 |
class ProtectedError(IntegrityError): |
| 11 |
def __init__(self, msg, protected_objects): |
| 12 |
self.protected_objects = protected_objects |
| 13 |
super().__init__(msg, protected_objects) |
| 14 |
|
| 15 |
|
| 16 |
class RestrictedError(IntegrityError): |
| 17 |
def __init__(self, msg, restricted_objects): |
| 18 |
self.restricted_objects = restricted_objects |
| 19 |
super().__init__(msg, restricted_objects) |
| 20 |
|
| 21 |
|
| 22 |
def CASCADE(collector, field, sub_objs, using): |
| 23 |
collector.collect( |
| 24 |
sub_objs, |
| 25 |
source=field.remote_field.model, |
| 26 |
source_attr=field.name, |
| 27 |
nullable=field.null, |
| 28 |
fail_on_restricted=False, |
| 29 |
) |
| 30 |
if field.null and not connections[using].features.can_defer_constraint_checks: |
| 31 |
collector.add_field_update(field, None, sub_objs) |
| 32 |
|
| 33 |
|
| 34 |
def PROTECT(collector, field, sub_objs, using): |
| 35 |
raise ProtectedError( |
| 36 |
"Cannot delete some instances of model '%s' because they are " |
| 37 |
"referenced through a protected foreign key: '%s.%s'" |
| 38 |
% ( |
| 39 |
field.remote_field.model.__name__, |
| 40 |
sub_objs[0].__class__.__name__, |
| 41 |
field.name, |
| 42 |
), |
| 43 |
sub_objs, |
| 44 |
) |
| 45 |
|
| 46 |
|
| 47 |
def RESTRICT(collector, field, sub_objs, using): |
| 48 |
collector.add_restricted_objects(field, sub_objs) |
| 49 |
collector.add_dependency(field.remote_field.model, field.model) |
| 50 |
|
| 51 |
|
| 52 |
def SET(value): |
| 53 |
if callable(value): |
| 54 |
|
| 55 |
def set_on_delete(collector, field, sub_objs, using): |
| 56 |
collector.add_field_update(field, value(), sub_objs) |
| 57 |
|
| 58 |
else: |
| 59 |
|
| 60 |
def set_on_delete(collector, field, sub_objs, using): |
| 61 |
collector.add_field_update(field, value, sub_objs) |
| 62 |
|
| 63 |
set_on_delete.lazy_sub_objs = True |
| 64 |
|
| 65 |
set_on_delete.deconstruct = lambda: ("django.db.models.SET", (value,), {}) |
| 66 |
return set_on_delete |
| 67 |
|
| 68 |
|
| 69 |
def SET_NULL(collector, field, sub_objs, using): |
| 70 |
collector.add_field_update(field, None, sub_objs) |
| 71 |
|
| 72 |
|
| 73 |
SET_NULL.lazy_sub_objs = True |
| 74 |
|
| 75 |
|
| 76 |
def SET_DEFAULT(collector, field, sub_objs, using): |
| 77 |
collector.add_field_update(field, field.get_default(), sub_objs) |
| 78 |
|
| 79 |
|
| 80 |
def DO_NOTHING(collector, field, sub_objs, using): |
| 81 |
pass |
| 82 |
|
| 83 |
|
| 84 |
def get_candidate_relations_to_delete(opts): |
| 85 |
# The candidate relations are the ones that come from N-1 and 1-1 relations. |
| 86 |
# N-N (i.e., many-to-many) relations aren't candidates for deletion. |
| 87 |
return ( |
| 88 |
f |
| 89 |
for f in opts.get_fields(include_hidden=True) |
| 90 |
if f.auto_created and not f.concrete and (f.one_to_one or f.one_to_many) |
| 91 |
) |
| 92 |
|
| 93 |
|
| 94 |
class Collector: |
| 95 |
def __init__(self, using, origin=None): |
| 96 |
self.using = using |
| 97 |
# A Model or QuerySet object. |
| 98 |
self.origin = origin |
| 99 |
# Initially, {model: {instances}}, later values become lists. |
| 100 |
self.data = defaultdict(set) |
| 101 |
# {(field, value): [instances, …]} |
| 102 |
self.field_updates = defaultdict(list) |
| 103 |
# {model: {field: {instances}}} |
| 104 |
self.restricted_objects = defaultdict(partial(defaultdict, set)) |
| 105 |
# fast_deletes is a list of queryset-likes that can be deleted without |
| 106 |
# fetching the objects into memory. |
| 107 |
self.fast_deletes = [] |
| 108 |
|
| 109 |
# Tracks deletion-order dependency for databases without transactions |
| 110 |
# or ability to defer constraint checks. Only concrete model classes |
| 111 |
# should be included, as the dependencies exist only between actual |
| 112 |
# database tables; proxy models are represented here by their concrete |
| 113 |
# parent. |
| 114 |
self.dependencies = defaultdict(set) # {model: {models}} |
| 115 |
|
| 116 |
def add(self, objs, source=None, nullable=False, reverse_dependency=False): |
| 117 |
""" |
| 118 |
Add 'objs' to the collection of objects to be deleted. If the call is |
| 119 |
the result of a cascade, 'source' should be the model that caused it, |
| 120 |
and 'nullable' should be set to True if the relation can be null. |
| 121 |
|
| 122 |
Return a list of all objects that were not already collected. |
| 123 |
""" |
| 124 |
if not objs: |
| 125 |
return [] |
| 126 |
new_objs = [] |
| 127 |
model = objs[0].__class__ |
| 128 |
instances = self.data[model] |
| 129 |
for obj in objs: |
| 130 |
if obj not in instances: |
| 131 |
new_objs.append(obj) |
| 132 |
instances.update(new_objs) |
| 133 |
# Nullable relationships can be ignored -- they are nulled out before |
| 134 |
# deleting, and therefore do not affect the order in which objects have |
| 135 |
# to be deleted. |
| 136 |
if source is not None and not nullable: |
| 137 |
self.add_dependency(source, model, reverse_dependency=reverse_dependency) |
| 138 |
return new_objs |
| 139 |
|
| 140 |
def add_dependency(self, model, dependency, reverse_dependency=False): |
| 141 |
if reverse_dependency: |
| 142 |
model, dependency = dependency, model |
| 143 |
self.dependencies[model._meta.concrete_model].add( |
| 144 |
dependency._meta.concrete_model |
| 145 |
) |
| 146 |
self.data.setdefault(dependency, self.data.default_factory()) |
| 147 |
|
| 148 |
def add_field_update(self, field, value, objs): |
| 149 |
""" |
| 150 |
Schedule a field update. 'objs' must be a homogeneous iterable |
| 151 |
collection of model instances (e.g. a QuerySet). |
| 152 |
""" |
| 153 |
self.field_updates[field, value].append(objs) |
| 154 |
|
| 155 |
def add_restricted_objects(self, field, objs): |
| 156 |
if objs: |
| 157 |
model = objs[0].__class__ |
| 158 |
self.restricted_objects[model][field].update(objs) |
| 159 |
|
| 160 |
def clear_restricted_objects_from_set(self, model, objs): |
| 161 |
if model in self.restricted_objects: |
| 162 |
self.restricted_objects[model] = { |
| 163 |
field: items - objs |
| 164 |
for field, items in self.restricted_objects[model].items() |
| 165 |
} |
| 166 |
|
| 167 |
def clear_restricted_objects_from_queryset(self, model, qs): |
| 168 |
if model in self.restricted_objects: |
| 169 |
objs = set( |
| 170 |
qs.filter( |
| 171 |
pk__in=[ |
| 172 |
obj.pk |
| 173 |
for objs in self.restricted_objects[model].values() |
| 174 |
for obj in objs |
| 175 |
] |
| 176 |
) |
| 177 |
) |
| 178 |
self.clear_restricted_objects_from_set(model, objs) |
| 179 |
|
| 180 |
def _has_signal_listeners(self, model): |
| 181 |
return signals.pre_delete.has_listeners( |
| 182 |
model |
| 183 |
) or signals.post_delete.has_listeners(model) |
| 184 |
|
| 185 |
def can_fast_delete(self, objs, from_field=None): |
| 186 |
""" |
| 187 |
Determine if the objects in the given queryset-like or single object |
| 188 |
can be fast-deleted. This can be done if there are no cascades, no |
| 189 |
parents and no signal listeners for the object class. |
| 190 |
|
| 191 |
The 'from_field' tells where we are coming from - we need this to |
| 192 |
determine if the objects are in fact to be deleted. Allow also |
| 193 |
skipping parent -> child -> parent chain preventing fast delete of |
| 194 |
the child. |
| 195 |
""" |
| 196 |
if from_field and from_field.remote_field.on_delete is not CASCADE: |
| 197 |
return False |
| 198 |
if hasattr(objs, "_meta"): |
| 199 |
model = objs._meta.model |
| 200 |
elif hasattr(objs, "model") and hasattr(objs, "_raw_delete"): |
| 201 |
model = objs.model |
| 202 |
else: |
| 203 |
return False |
| 204 |
if self._has_signal_listeners(model): |
| 205 |
return False |
| 206 |
# The use of from_field comes from the need to avoid cascade back to |
| 207 |
# parent when parent delete is cascading to child. |
| 208 |
opts = model._meta |
| 209 |
return ( |
| 210 |
all( |
| 211 |
link == from_field |
| 212 |
for link in opts.concrete_model._meta.parents.values() |
| 213 |
) |
| 214 |
and |
| 215 |
# Foreign keys pointing to this model. |
| 216 |
all( |
| 217 |
related.field.remote_field.on_delete is DO_NOTHING |
| 218 |
for related in get_candidate_relations_to_delete(opts) |
| 219 |
) |
| 220 |
and ( |
| 221 |
# Something like generic foreign key. |
| 222 |
not any( |
| 223 |
hasattr(field, "bulk_related_objects") |
| 224 |
for field in opts.private_fields |
| 225 |
) |
| 226 |
) |
| 227 |
) |
| 228 |
|
| 229 |
def get_del_batches(self, objs, fields): |
| 230 |
""" |
| 231 |
Return the objs in suitably sized batches for the used connection. |
| 232 |
""" |
| 233 |
conn_batch_size = max( |
| 234 |
connections[self.using].ops.bulk_batch_size(fields, objs), 1 |
| 235 |
) |
| 236 |
if len(objs) > conn_batch_size: |
| 237 |
return [ |
| 238 |
objs[i : i + conn_batch_size] |
| 239 |
for i in range(0, len(objs), conn_batch_size) |
| 240 |
] |
| 241 |
else: |
| 242 |
return [objs] |
| 243 |
|
| 244 |
def collect( |
| 245 |
self, |
| 246 |
objs, |
| 247 |
source=None, |
| 248 |
nullable=False, |
| 249 |
collect_related=True, |
| 250 |
source_attr=None, |
| 251 |
reverse_dependency=False, |
| 252 |
keep_parents=False, |
| 253 |
fail_on_restricted=True, |
| 254 |
): |
| 255 |
""" |
| 256 |
Add 'objs' to the collection of objects to be deleted as well as all |
| 257 |
parent instances. 'objs' must be a homogeneous iterable collection of |
| 258 |
model instances (e.g. a QuerySet). If 'collect_related' is True, |
| 259 |
related objects will be handled by their respective on_delete handler. |
| 260 |
|
| 261 |
If the call is the result of a cascade, 'source' should be the model |
| 262 |
that caused it and 'nullable' should be set to True, if the relation |
| 263 |
can be null. |
| 264 |
|
| 265 |
If 'reverse_dependency' is True, 'source' will be deleted before the |
| 266 |
current model, rather than after. (Needed for cascading to parent |
| 267 |
models, the one case in which the cascade follows the forwards |
| 268 |
direction of an FK rather than the reverse direction.) |
| 269 |
|
| 270 |
If 'keep_parents' is True, data of parent model's will be not deleted. |
| 271 |
|
| 272 |
If 'fail_on_restricted' is False, error won't be raised even if it's |
| 273 |
prohibited to delete such objects due to RESTRICT, that defers |
| 274 |
restricted object checking in recursive calls where the top-level call |
| 275 |
may need to collect more objects to determine whether restricted ones |
| 276 |
can be deleted. |
| 277 |
""" |
| 278 |
if self.can_fast_delete(objs): |
| 279 |
self.fast_deletes.append(objs) |
| 280 |
return |
| 281 |
new_objs = self.add( |
| 282 |
objs, source, nullable, reverse_dependency=reverse_dependency |
| 283 |
) |
| 284 |
if not new_objs: |
| 285 |
return |
| 286 |
|
| 287 |
model = new_objs[0].__class__ |
| 288 |
|
| 289 |
if not keep_parents: |
| 290 |
# Recursively collect concrete model's parent models, but not their |
| 291 |
# related objects. These will be found by meta.get_fields() |
| 292 |
concrete_model = model._meta.concrete_model |
| 293 |
for ptr in concrete_model._meta.parents.values(): |
| 294 |
if ptr: |
| 295 |
parent_objs = [getattr(obj, ptr.name) for obj in new_objs] |
| 296 |
self.collect( |
| 297 |
parent_objs, |
| 298 |
source=model, |
| 299 |
source_attr=ptr.remote_field.related_name, |
| 300 |
collect_related=False, |
| 301 |
reverse_dependency=True, |
| 302 |
fail_on_restricted=False, |
| 303 |
) |
| 304 |
if not collect_related: |
| 305 |
return |
| 306 |
|
| 307 |
model_fast_deletes = defaultdict(list) |
| 308 |
protected_objects = defaultdict(list) |
| 309 |
for related in get_candidate_relations_to_delete(model._meta): |
| 310 |
# Preserve parent reverse relationships if keep_parents=True. |
| 311 |
if keep_parents and related.model in model._meta.all_parents: |
| 312 |
continue |
| 313 |
field = related.field |
| 314 |
on_delete = field.remote_field.on_delete |
| 315 |
if on_delete == DO_NOTHING: |
| 316 |
continue |
| 317 |
related_model = related.related_model |
| 318 |
if self.can_fast_delete(related_model, from_field=field): |
| 319 |
model_fast_deletes[related_model].append(field) |
| 320 |
continue |
| 321 |
batches = self.get_del_batches(new_objs, [field]) |
| 322 |
for batch in batches: |
| 323 |
sub_objs = self.related_objects(related_model, [field], batch) |
| 324 |
# Non-referenced fields can be deferred if no signal receivers |
| 325 |
# are connected for the related model as they'll never be |
| 326 |
# exposed to the user. Skip field deferring when some |
| 327 |
# relationships are select_related as interactions between both |
| 328 |
# features are hard to get right. This should only happen in |
| 329 |
# the rare cases where .related_objects is overridden anyway. |
| 330 |
if not ( |
| 331 |
sub_objs.query.select_related |
| 332 |
or self._has_signal_listeners(related_model) |
| 333 |
): |
| 334 |
referenced_fields = set( |
| 335 |
chain.from_iterable( |
| 336 |
(rf.attname for rf in rel.field.foreign_related_fields) |
| 337 |
for rel in get_candidate_relations_to_delete( |
| 338 |
related_model._meta |
| 339 |
) |
| 340 |
) |
| 341 |
) |
| 342 |
sub_objs = sub_objs.only(*tuple(referenced_fields)) |
| 343 |
if getattr(on_delete, "lazy_sub_objs", False) or sub_objs: |
| 344 |
try: |
| 345 |
on_delete(self, field, sub_objs, self.using) |
| 346 |
except ProtectedError as error: |
| 347 |
key = "'%s.%s'" % (field.model.__name__, field.name) |
| 348 |
protected_objects[key] += error.protected_objects |
| 349 |
if protected_objects: |
| 350 |
raise ProtectedError( |
| 351 |
"Cannot delete some instances of model %r because they are " |
| 352 |
"referenced through protected foreign keys: %s." |
| 353 |
% ( |
| 354 |
model.__name__, |
| 355 |
", ".join(protected_objects), |
| 356 |
), |
| 357 |
set(chain.from_iterable(protected_objects.values())), |
| 358 |
) |
| 359 |
for related_model, related_fields in model_fast_deletes.items(): |
| 360 |
batches = self.get_del_batches(new_objs, related_fields) |
| 361 |
for batch in batches: |
| 362 |
sub_objs = self.related_objects(related_model, related_fields, batch) |
| 363 |
self.fast_deletes.append(sub_objs) |
| 364 |
for field in model._meta.private_fields: |
| 365 |
if hasattr(field, "bulk_related_objects"): |
| 366 |
# It's something like generic foreign key. |
| 367 |
sub_objs = field.bulk_related_objects(new_objs, self.using) |
| 368 |
self.collect( |
| 369 |
sub_objs, source=model, nullable=True, fail_on_restricted=False |
| 370 |
) |
| 371 |
|
| 372 |
if fail_on_restricted: |
| 373 |
# Raise an error if collected restricted objects (RESTRICT) aren't |
| 374 |
# candidates for deletion also collected via CASCADE. |
| 375 |
for related_model, instances in self.data.items(): |
| 376 |
self.clear_restricted_objects_from_set(related_model, instances) |
| 377 |
for qs in self.fast_deletes: |
| 378 |
self.clear_restricted_objects_from_queryset(qs.model, qs) |
| 379 |
if self.restricted_objects.values(): |
| 380 |
restricted_objects = defaultdict(list) |
| 381 |
for related_model, fields in self.restricted_objects.items(): |
| 382 |
for field, objs in fields.items(): |
| 383 |
if objs: |
| 384 |
key = "'%s.%s'" % (related_model.__name__, field.name) |
| 385 |
restricted_objects[key] += objs |
| 386 |
if restricted_objects: |
| 387 |
raise RestrictedError( |
| 388 |
"Cannot delete some instances of model %r because " |
| 389 |
"they are referenced through restricted foreign keys: " |
| 390 |
"%s." |
| 391 |
% ( |
| 392 |
model.__name__, |
| 393 |
", ".join(restricted_objects), |
| 394 |
), |
| 395 |
set(chain.from_iterable(restricted_objects.values())), |
| 396 |
) |
| 397 |
|
| 398 |
def related_objects(self, related_model, related_fields, objs): |
| 399 |
""" |
| 400 |
Get a QuerySet of the related model to objs via related fields. |
| 401 |
""" |
| 402 |
predicate = query_utils.Q.create( |
| 403 |
[(f"{related_field.name}__in", objs) for related_field in related_fields], |
| 404 |
connector=query_utils.Q.OR, |
| 405 |
) |
| 406 |
return related_model._base_manager.using(self.using).filter(predicate) |
| 407 |
|
| 408 |
def instances_with_model(self): |
| 409 |
for model, instances in self.data.items(): |
| 410 |
for obj in instances: |
| 411 |
yield model, obj |
| 412 |
|
| 413 |
def sort(self): |
| 414 |
sorted_models = [] |
| 415 |
concrete_models = set() |
| 416 |
models = list(self.data) |
| 417 |
while len(sorted_models) < len(models): |
| 418 |
found = False |
| 419 |
for model in models: |
| 420 |
if model in sorted_models: |
| 421 |
continue |
| 422 |
dependencies = self.dependencies.get(model._meta.concrete_model) |
| 423 |
if not (dependencies and dependencies.difference(concrete_models)): |
| 424 |
sorted_models.append(model) |
| 425 |
concrete_models.add(model._meta.concrete_model) |
| 426 |
found = True |
| 427 |
if not found: |
| 428 |
return |
| 429 |
self.data = {model: self.data[model] for model in sorted_models} |
| 430 |
|
| 431 |
def delete(self): |
| 432 |
# sort instance collections |
| 433 |
for model, instances in self.data.items(): |
| 434 |
self.data[model] = sorted(instances, key=attrgetter("pk")) |
| 435 |
|
| 436 |
# if possible, bring the models in an order suitable for databases that |
| 437 |
# don't support transactions or cannot defer constraint checks until the |
| 438 |
# end of a transaction. |
| 439 |
self.sort() |
| 440 |
# number of objects deleted for each model label |
| 441 |
deleted_counter = Counter() |
| 442 |
|
| 443 |
# Optimize for the case with a single obj and no dependencies |
| 444 |
if len(self.data) == 1 and len(instances) == 1: |
| 445 |
instance = list(instances)[0] |
| 446 |
if self.can_fast_delete(instance): |
| 447 |
with transaction.mark_for_rollback_on_error(self.using): |
| 448 |
count = sql.DeleteQuery(model).delete_batch( |
| 449 |
[instance.pk], self.using |
| 450 |
) |
| 451 |
setattr(instance, model._meta.pk.attname, None) |
| 452 |
return count, {model._meta.label: count} |
| 453 |
|
| 454 |
with transaction.atomic(using=self.using, savepoint=False): |
| 455 |
# send pre_delete signals |
| 456 |
for model, obj in self.instances_with_model(): |
| 457 |
if not model._meta.auto_created: |
| 458 |
signals.pre_delete.send( |
| 459 |
sender=model, |
| 460 |
instance=obj, |
| 461 |
using=self.using, |
| 462 |
origin=self.origin, |
| 463 |
) |
| 464 |
|
| 465 |
# fast deletes |
| 466 |
for qs in self.fast_deletes: |
| 467 |
count = qs._raw_delete(using=self.using) |
| 468 |
if count: |
| 469 |
deleted_counter[qs.model._meta.label] += count |
| 470 |
|
| 471 |
# update fields |
| 472 |
for (field, value), instances_list in self.field_updates.items(): |
| 473 |
updates = [] |
| 474 |
objs = [] |
| 475 |
for instances in instances_list: |
| 476 |
if ( |
| 477 |
isinstance(instances, models.QuerySet) |
| 478 |
and instances._result_cache is None |
| 479 |
): |
| 480 |
updates.append(instances) |
| 481 |
else: |
| 482 |
objs.extend(instances) |
| 483 |
if updates: |
| 484 |
combined_updates = reduce(or_, updates) |
| 485 |
combined_updates.update(**{field.name: value}) |
| 486 |
if objs: |
| 487 |
model = objs[0].__class__ |
| 488 |
query = sql.UpdateQuery(model) |
| 489 |
query.update_batch( |
| 490 |
list({obj.pk for obj in objs}), {field.name: value}, self.using |
| 491 |
) |
| 492 |
|
| 493 |
# reverse instance collections |
| 494 |
for instances in self.data.values(): |
| 495 |
instances.reverse() |
| 496 |
|
| 497 |
# delete instances |
| 498 |
for model, instances in self.data.items(): |
| 499 |
query = sql.DeleteQuery(model) |
| 500 |
pk_list = [obj.pk for obj in instances] |
| 501 |
count = query.delete_batch(pk_list, self.using) |
| 502 |
if count: |
| 503 |
deleted_counter[model._meta.label] += count |
| 504 |
|
| 505 |
if not model._meta.auto_created: |
| 506 |
for obj in instances: |
| 507 |
signals.post_delete.send( |
| 508 |
sender=model, |
| 509 |
instance=obj, |
| 510 |
using=self.using, |
| 511 |
origin=self.origin, |
| 512 |
) |
| 513 |
|
| 514 |
for model, instances in self.data.items(): |
| 515 |
for instance in instances: |
| 516 |
setattr(instance, model._meta.pk.attname, None) |
| 517 |
return sum(deleted_counter.values()), dict(deleted_counter) |