Debian Codegraph @

django/db/models/query_utils.py

raw
1 """
2 Various data structures used in query construction.
3
4 Factored out from django.db.models.query to avoid making the main module very
5 large and/or so that they can be used by other modules without getting into
6 circular import difficulties.
7 """
8
9 import functools
10 import inspect
11 import logging
12 from collections import namedtuple
13 from contextlib import nullcontext
14
15 from django.core.exceptions import FieldError
16 from django.db import DEFAULT_DB_ALIAS, DatabaseError, connections, models, transaction
17 from django.db.models.constants import LOOKUP_SEP
18 from django.utils import tree
19 from django.utils.functional import cached_property
20 from django.utils.hashable import make_hashable
21
22 logger = logging.getLogger("django.db.models")
23
24 # PathInfo is used when converting lookups (fk__somecol). The contents
25 # describe the relation in Model terms (model Options and Fields for both
26 # sides of the relation. The join_field is the field backing the relation.
27 PathInfo = namedtuple(
28 "PathInfo",
29 "from_opts to_opts target_fields join_field m2m direct filtered_relation",
30 )
31
32
33 def subclasses(cls):
34 yield cls
35 for subclass in cls.__subclasses__():
36 yield from subclasses(subclass)
37
38
39 class Q(tree.Node):
40 """
41 Encapsulate filters as objects that can then be combined logically (using
42 `&` and `|`).
43 """
44
45 # Connection types
46 AND = "AND"
47 OR = "OR"
48 XOR = "XOR"
49 default = AND
50 conditional = True
51 connectors = (None, AND, OR, XOR)
52
53 def __init__(self, *args, _connector=None, _negated=False, **kwargs):
54 if _connector not in self.connectors:
55 connector_reprs = ", ".join(f"{conn!r}" for conn in self.connectors[1:])
56 raise ValueError(f"_connector must be one of {connector_reprs}, or None.")
57 super().__init__(
58 children=[*args, *sorted(kwargs.items())],
59 connector=_connector,
60 negated=_negated,
61 )
62
63 def _combine(self, other, conn):
64 if getattr(other, "conditional", False) is False:
65 raise TypeError(other)
66 if not self:
67 return other.copy()
68 if not other and isinstance(other, Q):
69 return self.copy()
70
71 obj = self.create(connector=conn)
72 obj.add(self, conn)
73 obj.add(other, conn)
74 return obj
75
76 def __or__(self, other):
77 return self._combine(other, self.OR)
78
79 def __and__(self, other):
80 return self._combine(other, self.AND)
81
82 def __xor__(self, other):
83 return self._combine(other, self.XOR)
84
85 def __invert__(self):
86 obj = self.copy()
87 obj.negate()
88 return obj
89
90 def resolve_expression(
91 self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False
92 ):
93 # We must promote any new joins to left outer joins so that when Q is
94 # used as an expression, rows aren't filtered due to joins.
95 clause, joins = query._add_q(
96 self,
97 reuse,
98 allow_joins=allow_joins,
99 split_subq=False,
100 check_filterable=False,
101 summarize=summarize,
102 )
103 query.promote_joins(joins)
104 return clause
105
106 def replace_expressions(self, replacements):
107 if not replacements:
108 return self
109 clone = self.create(connector=self.connector, negated=self.negated)
110 for child in self.children:
111 child_replacement = child
112 if isinstance(child, tuple):
113 lhs, rhs = child
114 if LOOKUP_SEP in lhs:
115 path, lookup = lhs.rsplit(LOOKUP_SEP, 1)
116 else:
117 path = lhs
118 lookup = None
119 field = models.F(path)
120 if (
121 field_replacement := field.replace_expressions(replacements)
122 ) is not field:
123 # Handle the implicit __exact case by falling back to an
124 # extra transform when get_lookup returns no match for the
125 # last component of the path.
126 if lookup is None:
127 lookup = "exact"
128 if (lookup_class := field_replacement.get_lookup(lookup)) is None:
129 if (
130 transform_class := field_replacement.get_transform(lookup)
131 ) is not None:
132 field_replacement = transform_class(field_replacement)
133 lookup = "exact"
134 lookup_class = field_replacement.get_lookup(lookup)
135 if rhs is None and lookup == "exact":
136 lookup_class = field_replacement.get_lookup("isnull")
137 rhs = True
138 if lookup_class is not None:
139 child_replacement = lookup_class(field_replacement, rhs)
140 else:
141 child_replacement = child.replace_expressions(replacements)
142 clone.children.append(child_replacement)
143 return clone
144
145 def flatten(self):
146 """
147 Recursively yield this Q object and all subexpressions, in depth-first
148 order.
149 """
150 yield self
151 for child in self.children:
152 if isinstance(child, tuple):
153 # Use the lookup.
154 child = child[1]
155 if hasattr(child, "flatten"):
156 yield from child.flatten()
157 else:
158 yield child
159
160 def check(self, against, using=DEFAULT_DB_ALIAS):
161 """
162 Do a database query to check if the expressions of the Q instance
163 matches against the expressions.
164 """
165 # Avoid circular imports.
166 from django.db.models import BooleanField, Value
167 from django.db.models.functions import Coalesce
168 from django.db.models.sql import Query
169 from django.db.models.sql.constants import SINGLE
170
171 query = Query(None)
172 for name, value in against.items():
173 if not hasattr(value, "resolve_expression"):
174 value = Value(value)
175 query.add_annotation(value, name, select=False)
176 query.add_annotation(Value(1), "_check")
177 connection = connections[using]
178 # This will raise a FieldError if a field is missing in "against".
179 if connection.features.supports_comparing_boolean_expr:
180 query.add_q(Q(Coalesce(self, True, output_field=BooleanField())))
181 else:
182 query.add_q(self)
183 compiler = query.get_compiler(using=using)
184 context_manager = (
185 transaction.atomic(using=using)
186 if connection.in_atomic_block
187 else nullcontext()
188 )
189 try:
190 with context_manager:
191 return compiler.execute_sql(SINGLE) is not None
192 except DatabaseError as e:
193 logger.warning("Got a database error calling check() on %r: %s", self, e)
194 return True
195
196 def deconstruct(self):
197 path = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
198 if path.startswith("django.db.models.query_utils"):
199 path = path.replace("django.db.models.query_utils", "django.db.models")
200 args = tuple(self.children)
201 kwargs = {}
202 if self.connector != self.default:
203 kwargs["_connector"] = self.connector
204 if self.negated:
205 kwargs["_negated"] = True
206 return path, args, kwargs
207
208 @cached_property
209 def identity(self):
210 path, args, kwargs = self.deconstruct()
211 identity = [path, *kwargs.items()]
212 for child in args:
213 if isinstance(child, tuple):
214 arg, value = child
215 value = make_hashable(value)
216 identity.append((arg, value))
217 else:
218 identity.append(child)
219 return tuple(identity)
220
221 def __eq__(self, other):
222 if not isinstance(other, Q):
223 return NotImplemented
224 return other.identity == self.identity
225
226 def __hash__(self):
227 return hash(self.identity)
228
229 @cached_property
230 def referenced_base_fields(self):
231 """
232 Retrieve all base fields referenced directly or through F expressions
233 excluding any fields referenced through joins.
234 """
235 # Avoid circular imports.
236 from django.db.models.sql import query
237
238 return {
239 child.split(LOOKUP_SEP, 1)[0] for child in query.get_children_from_q(self)
240 }
241
242
243 class DeferredAttribute:
244 """
245 A wrapper for a deferred-loading field. When the value is read from this
246 object the first time, the query is executed.
247 """
248
249 def __init__(self, field):
250 self.field = field
251
252 def __get__(self, instance, cls=None):
253 """
254 Retrieve and caches the value from the datastore on the first lookup.
255 Return the cached value.
256 """
257 if instance is None:
258 return self
259 data = instance.__dict__
260 field_name = self.field.attname
261 if field_name not in data:
262 # Let's see if the field is part of the parent chain. If so we
263 # might be able to reuse the already loaded value. Refs #18343.
264 val = self._check_parent_chain(instance)
265 if val is None:
266 if not instance._is_pk_set() and self.field.generated:
267 raise AttributeError(
268 "Cannot read a generated field from an unsaved model."
269 )
270 instance.refresh_from_db(fields=[field_name])
271 else:
272 data[field_name] = val
273 return data[field_name]
274
275 def _check_parent_chain(self, instance):
276 """
277 Check if the field value can be fetched from a parent field already
278 loaded in the instance. This can be done if the to-be fetched
279 field is a primary key field.
280 """
281 opts = instance._meta
282 link_field = opts.get_ancestor_link(self.field.model)
283 if self.field.primary_key and self.field != link_field:
284 return getattr(instance, link_field.attname)
285 return None
286
287
288 class class_or_instance_method:
289 """
290 Hook used in RegisterLookupMixin to return partial functions depending on
291 the caller type (instance or class of models.Field).
292 """
293
294 def __init__(self, class_method, instance_method):
295 self.class_method = class_method
296 self.instance_method = instance_method
297
298 def __get__(self, instance, owner):
299 if instance is None:
300 return functools.partial(self.class_method, owner)
301 return functools.partial(self.instance_method, instance)
302
303
304 class RegisterLookupMixin:
305 def _get_lookup(self, lookup_name):
306 return self.get_lookups().get(lookup_name, None)
307
308 @functools.cache
309 def get_class_lookups(cls):
310 class_lookups = [
311 parent.__dict__.get("class_lookups", {}) for parent in inspect.getmro(cls)
312 ]
313 return cls.merge_dicts(class_lookups)
314
315 def get_instance_lookups(self):
316 class_lookups = self.get_class_lookups()
317 if instance_lookups := getattr(self, "instance_lookups", None):
318 return {**class_lookups, **instance_lookups}
319 return class_lookups
320
321 get_lookups = class_or_instance_method(get_class_lookups, get_instance_lookups)
322 get_class_lookups = classmethod(get_class_lookups)
323
324 def get_lookup(self, lookup_name):
325 from django.db.models.lookups import Lookup
326
327 found = self._get_lookup(lookup_name)
328 if found is None and hasattr(self, "output_field"):
329 return self.output_field.get_lookup(lookup_name)
330 if found is not None and not issubclass(found, Lookup):
331 return None
332 return found
333
334 def get_transform(self, lookup_name):
335 from django.db.models.lookups import Transform
336
337 found = self._get_lookup(lookup_name)
338 if found is None and hasattr(self, "output_field"):
339 return self.output_field.get_transform(lookup_name)
340 if found is not None and not issubclass(found, Transform):
341 return None
342 return found
343
344 @staticmethod
345 def merge_dicts(dicts):
346 """
347 Merge dicts in reverse to preference the order of the original list. e.g.,
348 merge_dicts([a, b]) will preference the keys in 'a' over those in 'b'.
349 """
350 merged = {}
351 for d in reversed(dicts):
352 merged.update(d)
353 return merged
354
355 @classmethod
356 def _clear_cached_class_lookups(cls):
357 for subclass in subclasses(cls):
358 subclass.get_class_lookups.cache_clear()
359
360 def register_class_lookup(cls, lookup, lookup_name=None):
361 if lookup_name is None:
362 lookup_name = lookup.lookup_name
363 if "class_lookups" not in cls.__dict__:
364 cls.class_lookups = {}
365 cls.class_lookups[lookup_name] = lookup
366 cls._clear_cached_class_lookups()
367 return lookup
368
369 def register_instance_lookup(self, lookup, lookup_name=None):
370 if lookup_name is None:
371 lookup_name = lookup.lookup_name
372 if "instance_lookups" not in self.__dict__:
373 self.instance_lookups = {}
374 self.instance_lookups[lookup_name] = lookup
375 return lookup
376
377 register_lookup = class_or_instance_method(
378 register_class_lookup, register_instance_lookup
379 )
380 register_class_lookup = classmethod(register_class_lookup)
381
382 def _unregister_class_lookup(cls, lookup, lookup_name=None):
383 """
384 Remove given lookup from cls lookups. For use in tests only as it's
385 not thread-safe.
386 """
387 if lookup_name is None:
388 lookup_name = lookup.lookup_name
389 del cls.class_lookups[lookup_name]
390 cls._clear_cached_class_lookups()
391
392 def _unregister_instance_lookup(self, lookup, lookup_name=None):
393 """
394 Remove given lookup from instance lookups. For use in tests only as
395 it's not thread-safe.
396 """
397 if lookup_name is None:
398 lookup_name = lookup.lookup_name
399 del self.instance_lookups[lookup_name]
400
401 _unregister_lookup = class_or_instance_method(
402 _unregister_class_lookup, _unregister_instance_lookup
403 )
404 _unregister_class_lookup = classmethod(_unregister_class_lookup)
405
406
407 def select_related_descend(field, restricted, requested, select_mask):
408 """
409 Return whether `field` should be used to descend deeper for
410 `select_related()` purposes.
411
412 Arguments:
413 * `field` - the field to be checked. Can be either a `Field` or
414 `ForeignObjectRel` instance.
415 * `restricted` - a boolean field, indicating if the field list has been
416 manually restricted using a select_related() clause.
417 * `requested` - the select_related() dictionary.
418 * `select_mask` - the dictionary of selected fields.
419 """
420 # Only relationships can be descended.
421 if not field.remote_field:
422 return False
423 # Forward MTI parent links should not be explicitly descended as they are
424 # always JOIN'ed against (unless excluded by `select_mask`).
425 if getattr(field.remote_field, "parent_link", False):
426 return False
427 # When `select_related()` is used without a `*requested` mask all
428 # relationships are descended unless they are nullable.
429 if not restricted:
430 return not field.null
431 # When `select_related(*requested)` is used only fields that are part of
432 # `requested` should be descended.
433 if field.name not in requested:
434 return False
435 # Prevent invalid usages of `select_related()` and `only()`/`defer()` such
436 # as `select_related("a").only("b")` and `select_related("a").defer("a")`.
437 if select_mask and field not in select_mask:
438 raise FieldError(
439 f"Field {field.model._meta.object_name}.{field.name} cannot be both "
440 "deferred and traversed using select_related at the same time."
441 )
442 return True
443
444
445 def refs_expression(lookup_parts, annotations):
446 """
447 Check if the lookup_parts contains references to the given annotations set.
448 Because the LOOKUP_SEP is contained in the default annotation names, check
449 each prefix of the lookup_parts for a match.
450 """
451 for n in range(1, len(lookup_parts) + 1):
452 level_n_lookup = LOOKUP_SEP.join(lookup_parts[0:n])
453 if annotations.get(level_n_lookup):
454 return level_n_lookup, lookup_parts[n:]
455 return None, ()
456
457
458 def check_rel_lookup_compatibility(model, target_opts, field):
459 """
460 Check that self.model is compatible with target_opts. Compatibility
461 is OK if:
462 1) model and opts match (where proxy inheritance is removed)
463 2) model is parent of opts' model or the other way around
464 """
465
466 def check(opts):
467 return (
468 model._meta.concrete_model == opts.concrete_model
469 or opts.concrete_model in model._meta.all_parents
470 or model in opts.all_parents
471 )
472
473 # If the field is a primary key, then doing a query against the field's
474 # model is ok, too. Consider the case:
475 # class Restaurant(models.Model):
476 # place = OneToOneField(Place, primary_key=True):
477 # Restaurant.objects.filter(pk__in=Restaurant.objects.all()).
478 # If we didn't have the primary key check, then pk__in (== place__in) would
479 # give Place's opts as the target opts, but Restaurant isn't compatible
480 # with that. This logic applies only to primary keys, as when doing __in=qs,
481 # we are going to turn this into __in=qs.values('pk') later on.
482 return check(target_opts) or (
483 getattr(field, "primary_key", False) and check(field.model._meta)
484 )
485
486
487 class FilteredRelation:
488 """Specify custom filtering in the ON clause of SQL joins."""
489
490 def __init__(self, relation_name, *, condition=Q()):
491 if not relation_name:
492 raise ValueError("relation_name cannot be empty.")
493 self.relation_name = relation_name
494 self.alias = None
495 if not isinstance(condition, Q):
496 raise ValueError("condition argument must be a Q() instance.")
497 # .condition and .resolved_condition have to be stored independently
498 # as the former must remain unchanged for Join.__eq__ to remain stable
499 # and reusable even once their .filtered_relation are resolved.
500 self.condition = condition
501 self.resolved_condition = None
502
503 def __eq__(self, other):
504 if not isinstance(other, self.__class__):
505 return NotImplemented
506 return (
507 self.relation_name == other.relation_name
508 and self.alias == other.alias
509 and self.condition == other.condition
510 )
511
512 def clone(self):
513 clone = FilteredRelation(self.relation_name, condition=self.condition)
514 clone.alias = self.alias
515 if (resolved_condition := self.resolved_condition) is not None:
516 clone.resolved_condition = resolved_condition.clone()
517 return clone
518
519 def relabeled_clone(self, change_map):
520 clone = self.clone()
521 if resolved_condition := clone.resolved_condition:
522 clone.resolved_condition = resolved_condition.relabeled_clone(change_map)
523 return clone
524
525 def resolve_expression(self, query, reuse, *args, **kwargs):
526 clone = self.clone()
527 clone.resolved_condition = query.build_filter(
528 self.condition,
529 can_reuse=reuse,
530 allow_joins=True,
531 split_subq=False,
532 update_join_types=False,
533 )[0]
534 return clone
535
536 def as_sql(self, compiler, connection):
537 return compiler.compile(self.resolved_condition)