| 1 |
""" |
| 2 |
Various data structures used in query construction. |
| 3 |
|
| 4 |
Factored out from django.db.models.query to avoid making the main module very |
| 5 |
large and/or so that they can be used by other modules without getting into |
| 6 |
circular import difficulties. |
| 7 |
""" |
| 8 |
|
| 9 |
import functools |
| 10 |
import inspect |
| 11 |
import logging |
| 12 |
from collections import namedtuple |
| 13 |
from contextlib import nullcontext |
| 14 |
|
| 15 |
from django.core.exceptions import FieldError |
| 16 |
from django.db import DEFAULT_DB_ALIAS, DatabaseError, connections, models, transaction |
| 17 |
from django.db.models.constants import LOOKUP_SEP |
| 18 |
from django.utils import tree |
| 19 |
from django.utils.functional import cached_property |
| 20 |
from django.utils.hashable import make_hashable |
| 21 |
|
| 22 |
logger = logging.getLogger("django.db.models") |
| 23 |
|
| 24 |
# PathInfo is used when converting lookups (fk__somecol). The contents |
| 25 |
# describe the relation in Model terms (model Options and Fields for both |
| 26 |
# sides of the relation. The join_field is the field backing the relation. |
| 27 |
PathInfo = namedtuple( |
| 28 |
"PathInfo", |
| 29 |
"from_opts to_opts target_fields join_field m2m direct filtered_relation", |
| 30 |
) |
| 31 |
|
| 32 |
|
| 33 |
def subclasses(cls): |
| 34 |
yield cls |
| 35 |
for subclass in cls.__subclasses__(): |
| 36 |
yield from subclasses(subclass) |
| 37 |
|
| 38 |
|
| 39 |
class Q(tree.Node): |
| 40 |
""" |
| 41 |
Encapsulate filters as objects that can then be combined logically (using |
| 42 |
`&` and `|`). |
| 43 |
""" |
| 44 |
|
| 45 |
# Connection types |
| 46 |
AND = "AND" |
| 47 |
OR = "OR" |
| 48 |
XOR = "XOR" |
| 49 |
default = AND |
| 50 |
conditional = True |
| 51 |
connectors = (None, AND, OR, XOR) |
| 52 |
|
| 53 |
def __init__(self, *args, _connector=None, _negated=False, **kwargs): |
| 54 |
if _connector not in self.connectors: |
| 55 |
connector_reprs = ", ".join(f"{conn!r}" for conn in self.connectors[1:]) |
| 56 |
raise ValueError(f"_connector must be one of {connector_reprs}, or None.") |
| 57 |
super().__init__( |
| 58 |
children=[*args, *sorted(kwargs.items())], |
| 59 |
connector=_connector, |
| 60 |
negated=_negated, |
| 61 |
) |
| 62 |
|
| 63 |
def _combine(self, other, conn): |
| 64 |
if getattr(other, "conditional", False) is False: |
| 65 |
raise TypeError(other) |
| 66 |
if not self: |
| 67 |
return other.copy() |
| 68 |
if not other and isinstance(other, Q): |
| 69 |
return self.copy() |
| 70 |
|
| 71 |
obj = self.create(connector=conn) |
| 72 |
obj.add(self, conn) |
| 73 |
obj.add(other, conn) |
| 74 |
return obj |
| 75 |
|
| 76 |
def __or__(self, other): |
| 77 |
return self._combine(other, self.OR) |
| 78 |
|
| 79 |
def __and__(self, other): |
| 80 |
return self._combine(other, self.AND) |
| 81 |
|
| 82 |
def __xor__(self, other): |
| 83 |
return self._combine(other, self.XOR) |
| 84 |
|
| 85 |
def __invert__(self): |
| 86 |
obj = self.copy() |
| 87 |
obj.negate() |
| 88 |
return obj |
| 89 |
|
| 90 |
def resolve_expression( |
| 91 |
self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False |
| 92 |
): |
| 93 |
# We must promote any new joins to left outer joins so that when Q is |
| 94 |
# used as an expression, rows aren't filtered due to joins. |
| 95 |
clause, joins = query._add_q( |
| 96 |
self, |
| 97 |
reuse, |
| 98 |
allow_joins=allow_joins, |
| 99 |
split_subq=False, |
| 100 |
check_filterable=False, |
| 101 |
summarize=summarize, |
| 102 |
) |
| 103 |
query.promote_joins(joins) |
| 104 |
return clause |
| 105 |
|
| 106 |
def replace_expressions(self, replacements): |
| 107 |
if not replacements: |
| 108 |
return self |
| 109 |
clone = self.create(connector=self.connector, negated=self.negated) |
| 110 |
for child in self.children: |
| 111 |
child_replacement = child |
| 112 |
if isinstance(child, tuple): |
| 113 |
lhs, rhs = child |
| 114 |
if LOOKUP_SEP in lhs: |
| 115 |
path, lookup = lhs.rsplit(LOOKUP_SEP, 1) |
| 116 |
else: |
| 117 |
path = lhs |
| 118 |
lookup = None |
| 119 |
field = models.F(path) |
| 120 |
if ( |
| 121 |
field_replacement := field.replace_expressions(replacements) |
| 122 |
) is not field: |
| 123 |
# Handle the implicit __exact case by falling back to an |
| 124 |
# extra transform when get_lookup returns no match for the |
| 125 |
# last component of the path. |
| 126 |
if lookup is None: |
| 127 |
lookup = "exact" |
| 128 |
if (lookup_class := field_replacement.get_lookup(lookup)) is None: |
| 129 |
if ( |
| 130 |
transform_class := field_replacement.get_transform(lookup) |
| 131 |
) is not None: |
| 132 |
field_replacement = transform_class(field_replacement) |
| 133 |
lookup = "exact" |
| 134 |
lookup_class = field_replacement.get_lookup(lookup) |
| 135 |
if rhs is None and lookup == "exact": |
| 136 |
lookup_class = field_replacement.get_lookup("isnull") |
| 137 |
rhs = True |
| 138 |
if lookup_class is not None: |
| 139 |
child_replacement = lookup_class(field_replacement, rhs) |
| 140 |
else: |
| 141 |
child_replacement = child.replace_expressions(replacements) |
| 142 |
clone.children.append(child_replacement) |
| 143 |
return clone |
| 144 |
|
| 145 |
def flatten(self): |
| 146 |
""" |
| 147 |
Recursively yield this Q object and all subexpressions, in depth-first |
| 148 |
order. |
| 149 |
""" |
| 150 |
yield self |
| 151 |
for child in self.children: |
| 152 |
if isinstance(child, tuple): |
| 153 |
# Use the lookup. |
| 154 |
child = child[1] |
| 155 |
if hasattr(child, "flatten"): |
| 156 |
yield from child.flatten() |
| 157 |
else: |
| 158 |
yield child |
| 159 |
|
| 160 |
def check(self, against, using=DEFAULT_DB_ALIAS): |
| 161 |
""" |
| 162 |
Do a database query to check if the expressions of the Q instance |
| 163 |
matches against the expressions. |
| 164 |
""" |
| 165 |
# Avoid circular imports. |
| 166 |
from django.db.models import BooleanField, Value |
| 167 |
from django.db.models.functions import Coalesce |
| 168 |
from django.db.models.sql import Query |
| 169 |
from django.db.models.sql.constants import SINGLE |
| 170 |
|
| 171 |
query = Query(None) |
| 172 |
for name, value in against.items(): |
| 173 |
if not hasattr(value, "resolve_expression"): |
| 174 |
value = Value(value) |
| 175 |
query.add_annotation(value, name, select=False) |
| 176 |
query.add_annotation(Value(1), "_check") |
| 177 |
connection = connections[using] |
| 178 |
# This will raise a FieldError if a field is missing in "against". |
| 179 |
if connection.features.supports_comparing_boolean_expr: |
| 180 |
query.add_q(Q(Coalesce(self, True, output_field=BooleanField()))) |
| 181 |
else: |
| 182 |
query.add_q(self) |
| 183 |
compiler = query.get_compiler(using=using) |
| 184 |
context_manager = ( |
| 185 |
transaction.atomic(using=using) |
| 186 |
if connection.in_atomic_block |
| 187 |
else nullcontext() |
| 188 |
) |
| 189 |
try: |
| 190 |
with context_manager: |
| 191 |
return compiler.execute_sql(SINGLE) is not None |
| 192 |
except DatabaseError as e: |
| 193 |
logger.warning("Got a database error calling check() on %r: %s", self, e) |
| 194 |
return True |
| 195 |
|
| 196 |
def deconstruct(self): |
| 197 |
path = "%s.%s" % (self.__class__.__module__, self.__class__.__name__) |
| 198 |
if path.startswith("django.db.models.query_utils"): |
| 199 |
path = path.replace("django.db.models.query_utils", "django.db.models") |
| 200 |
args = tuple(self.children) |
| 201 |
kwargs = {} |
| 202 |
if self.connector != self.default: |
| 203 |
kwargs["_connector"] = self.connector |
| 204 |
if self.negated: |
| 205 |
kwargs["_negated"] = True |
| 206 |
return path, args, kwargs |
| 207 |
|
| 208 |
@cached_property |
| 209 |
def identity(self): |
| 210 |
path, args, kwargs = self.deconstruct() |
| 211 |
identity = [path, *kwargs.items()] |
| 212 |
for child in args: |
| 213 |
if isinstance(child, tuple): |
| 214 |
arg, value = child |
| 215 |
value = make_hashable(value) |
| 216 |
identity.append((arg, value)) |
| 217 |
else: |
| 218 |
identity.append(child) |
| 219 |
return tuple(identity) |
| 220 |
|
| 221 |
def __eq__(self, other): |
| 222 |
if not isinstance(other, Q): |
| 223 |
return NotImplemented |
| 224 |
return other.identity == self.identity |
| 225 |
|
| 226 |
def __hash__(self): |
| 227 |
return hash(self.identity) |
| 228 |
|
| 229 |
@cached_property |
| 230 |
def referenced_base_fields(self): |
| 231 |
""" |
| 232 |
Retrieve all base fields referenced directly or through F expressions |
| 233 |
excluding any fields referenced through joins. |
| 234 |
""" |
| 235 |
# Avoid circular imports. |
| 236 |
from django.db.models.sql import query |
| 237 |
|
| 238 |
return { |
| 239 |
child.split(LOOKUP_SEP, 1)[0] for child in query.get_children_from_q(self) |
| 240 |
} |
| 241 |
|
| 242 |
|
| 243 |
class DeferredAttribute: |
| 244 |
""" |
| 245 |
A wrapper for a deferred-loading field. When the value is read from this |
| 246 |
object the first time, the query is executed. |
| 247 |
""" |
| 248 |
|
| 249 |
def __init__(self, field): |
| 250 |
self.field = field |
| 251 |
|
| 252 |
def __get__(self, instance, cls=None): |
| 253 |
""" |
| 254 |
Retrieve and caches the value from the datastore on the first lookup. |
| 255 |
Return the cached value. |
| 256 |
""" |
| 257 |
if instance is None: |
| 258 |
return self |
| 259 |
data = instance.__dict__ |
| 260 |
field_name = self.field.attname |
| 261 |
if field_name not in data: |
| 262 |
# Let's see if the field is part of the parent chain. If so we |
| 263 |
# might be able to reuse the already loaded value. Refs #18343. |
| 264 |
val = self._check_parent_chain(instance) |
| 265 |
if val is None: |
| 266 |
if not instance._is_pk_set() and self.field.generated: |
| 267 |
raise AttributeError( |
| 268 |
"Cannot read a generated field from an unsaved model." |
| 269 |
) |
| 270 |
instance.refresh_from_db(fields=[field_name]) |
| 271 |
else: |
| 272 |
data[field_name] = val |
| 273 |
return data[field_name] |
| 274 |
|
| 275 |
def _check_parent_chain(self, instance): |
| 276 |
""" |
| 277 |
Check if the field value can be fetched from a parent field already |
| 278 |
loaded in the instance. This can be done if the to-be fetched |
| 279 |
field is a primary key field. |
| 280 |
""" |
| 281 |
opts = instance._meta |
| 282 |
link_field = opts.get_ancestor_link(self.field.model) |
| 283 |
if self.field.primary_key and self.field != link_field: |
| 284 |
return getattr(instance, link_field.attname) |
| 285 |
return None |
| 286 |
|
| 287 |
|
| 288 |
class class_or_instance_method: |
| 289 |
""" |
| 290 |
Hook used in RegisterLookupMixin to return partial functions depending on |
| 291 |
the caller type (instance or class of models.Field). |
| 292 |
""" |
| 293 |
|
| 294 |
def __init__(self, class_method, instance_method): |
| 295 |
self.class_method = class_method |
| 296 |
self.instance_method = instance_method |
| 297 |
|
| 298 |
def __get__(self, instance, owner): |
| 299 |
if instance is None: |
| 300 |
return functools.partial(self.class_method, owner) |
| 301 |
return functools.partial(self.instance_method, instance) |
| 302 |
|
| 303 |
|
| 304 |
class RegisterLookupMixin: |
| 305 |
def _get_lookup(self, lookup_name): |
| 306 |
return self.get_lookups().get(lookup_name, None) |
| 307 |
|
| 308 |
@functools.cache |
| 309 |
def get_class_lookups(cls): |
| 310 |
class_lookups = [ |
| 311 |
parent.__dict__.get("class_lookups", {}) for parent in inspect.getmro(cls) |
| 312 |
] |
| 313 |
return cls.merge_dicts(class_lookups) |
| 314 |
|
| 315 |
def get_instance_lookups(self): |
| 316 |
class_lookups = self.get_class_lookups() |
| 317 |
if instance_lookups := getattr(self, "instance_lookups", None): |
| 318 |
return {**class_lookups, **instance_lookups} |
| 319 |
return class_lookups |
| 320 |
|
| 321 |
get_lookups = class_or_instance_method(get_class_lookups, get_instance_lookups) |
| 322 |
get_class_lookups = classmethod(get_class_lookups) |
| 323 |
|
| 324 |
def get_lookup(self, lookup_name): |
| 325 |
from django.db.models.lookups import Lookup |
| 326 |
|
| 327 |
found = self._get_lookup(lookup_name) |
| 328 |
if found is None and hasattr(self, "output_field"): |
| 329 |
return self.output_field.get_lookup(lookup_name) |
| 330 |
if found is not None and not issubclass(found, Lookup): |
| 331 |
return None |
| 332 |
return found |
| 333 |
|
| 334 |
def get_transform(self, lookup_name): |
| 335 |
from django.db.models.lookups import Transform |
| 336 |
|
| 337 |
found = self._get_lookup(lookup_name) |
| 338 |
if found is None and hasattr(self, "output_field"): |
| 339 |
return self.output_field.get_transform(lookup_name) |
| 340 |
if found is not None and not issubclass(found, Transform): |
| 341 |
return None |
| 342 |
return found |
| 343 |
|
| 344 |
@staticmethod |
| 345 |
def merge_dicts(dicts): |
| 346 |
""" |
| 347 |
Merge dicts in reverse to preference the order of the original list. e.g., |
| 348 |
merge_dicts([a, b]) will preference the keys in 'a' over those in 'b'. |
| 349 |
""" |
| 350 |
merged = {} |
| 351 |
for d in reversed(dicts): |
| 352 |
merged.update(d) |
| 353 |
return merged |
| 354 |
|
| 355 |
@classmethod |
| 356 |
def _clear_cached_class_lookups(cls): |
| 357 |
for subclass in subclasses(cls): |
| 358 |
subclass.get_class_lookups.cache_clear() |
| 359 |
|
| 360 |
def register_class_lookup(cls, lookup, lookup_name=None): |
| 361 |
if lookup_name is None: |
| 362 |
lookup_name = lookup.lookup_name |
| 363 |
if "class_lookups" not in cls.__dict__: |
| 364 |
cls.class_lookups = {} |
| 365 |
cls.class_lookups[lookup_name] = lookup |
| 366 |
cls._clear_cached_class_lookups() |
| 367 |
return lookup |
| 368 |
|
| 369 |
def register_instance_lookup(self, lookup, lookup_name=None): |
| 370 |
if lookup_name is None: |
| 371 |
lookup_name = lookup.lookup_name |
| 372 |
if "instance_lookups" not in self.__dict__: |
| 373 |
self.instance_lookups = {} |
| 374 |
self.instance_lookups[lookup_name] = lookup |
| 375 |
return lookup |
| 376 |
|
| 377 |
register_lookup = class_or_instance_method( |
| 378 |
register_class_lookup, register_instance_lookup |
| 379 |
) |
| 380 |
register_class_lookup = classmethod(register_class_lookup) |
| 381 |
|
| 382 |
def _unregister_class_lookup(cls, lookup, lookup_name=None): |
| 383 |
""" |
| 384 |
Remove given lookup from cls lookups. For use in tests only as it's |
| 385 |
not thread-safe. |
| 386 |
""" |
| 387 |
if lookup_name is None: |
| 388 |
lookup_name = lookup.lookup_name |
| 389 |
del cls.class_lookups[lookup_name] |
| 390 |
cls._clear_cached_class_lookups() |
| 391 |
|
| 392 |
def _unregister_instance_lookup(self, lookup, lookup_name=None): |
| 393 |
""" |
| 394 |
Remove given lookup from instance lookups. For use in tests only as |
| 395 |
it's not thread-safe. |
| 396 |
""" |
| 397 |
if lookup_name is None: |
| 398 |
lookup_name = lookup.lookup_name |
| 399 |
del self.instance_lookups[lookup_name] |
| 400 |
|
| 401 |
_unregister_lookup = class_or_instance_method( |
| 402 |
_unregister_class_lookup, _unregister_instance_lookup |
| 403 |
) |
| 404 |
_unregister_class_lookup = classmethod(_unregister_class_lookup) |
| 405 |
|
| 406 |
|
| 407 |
def select_related_descend(field, restricted, requested, select_mask): |
| 408 |
""" |
| 409 |
Return whether `field` should be used to descend deeper for |
| 410 |
`select_related()` purposes. |
| 411 |
|
| 412 |
Arguments: |
| 413 |
* `field` - the field to be checked. Can be either a `Field` or |
| 414 |
`ForeignObjectRel` instance. |
| 415 |
* `restricted` - a boolean field, indicating if the field list has been |
| 416 |
manually restricted using a select_related() clause. |
| 417 |
* `requested` - the select_related() dictionary. |
| 418 |
* `select_mask` - the dictionary of selected fields. |
| 419 |
""" |
| 420 |
# Only relationships can be descended. |
| 421 |
if not field.remote_field: |
| 422 |
return False |
| 423 |
# Forward MTI parent links should not be explicitly descended as they are |
| 424 |
# always JOIN'ed against (unless excluded by `select_mask`). |
| 425 |
if getattr(field.remote_field, "parent_link", False): |
| 426 |
return False |
| 427 |
# When `select_related()` is used without a `*requested` mask all |
| 428 |
# relationships are descended unless they are nullable. |
| 429 |
if not restricted: |
| 430 |
return not field.null |
| 431 |
# When `select_related(*requested)` is used only fields that are part of |
| 432 |
# `requested` should be descended. |
| 433 |
if field.name not in requested: |
| 434 |
return False |
| 435 |
# Prevent invalid usages of `select_related()` and `only()`/`defer()` such |
| 436 |
# as `select_related("a").only("b")` and `select_related("a").defer("a")`. |
| 437 |
if select_mask and field not in select_mask: |
| 438 |
raise FieldError( |
| 439 |
f"Field {field.model._meta.object_name}.{field.name} cannot be both " |
| 440 |
"deferred and traversed using select_related at the same time." |
| 441 |
) |
| 442 |
return True |
| 443 |
|
| 444 |
|
| 445 |
def refs_expression(lookup_parts, annotations): |
| 446 |
""" |
| 447 |
Check if the lookup_parts contains references to the given annotations set. |
| 448 |
Because the LOOKUP_SEP is contained in the default annotation names, check |
| 449 |
each prefix of the lookup_parts for a match. |
| 450 |
""" |
| 451 |
for n in range(1, len(lookup_parts) + 1): |
| 452 |
level_n_lookup = LOOKUP_SEP.join(lookup_parts[0:n]) |
| 453 |
if annotations.get(level_n_lookup): |
| 454 |
return level_n_lookup, lookup_parts[n:] |
| 455 |
return None, () |
| 456 |
|
| 457 |
|
| 458 |
def check_rel_lookup_compatibility(model, target_opts, field): |
| 459 |
""" |
| 460 |
Check that self.model is compatible with target_opts. Compatibility |
| 461 |
is OK if: |
| 462 |
1) model and opts match (where proxy inheritance is removed) |
| 463 |
2) model is parent of opts' model or the other way around |
| 464 |
""" |
| 465 |
|
| 466 |
def check(opts): |
| 467 |
return ( |
| 468 |
model._meta.concrete_model == opts.concrete_model |
| 469 |
or opts.concrete_model in model._meta.all_parents |
| 470 |
or model in opts.all_parents |
| 471 |
) |
| 472 |
|
| 473 |
# If the field is a primary key, then doing a query against the field's |
| 474 |
# model is ok, too. Consider the case: |
| 475 |
# class Restaurant(models.Model): |
| 476 |
# place = OneToOneField(Place, primary_key=True): |
| 477 |
# Restaurant.objects.filter(pk__in=Restaurant.objects.all()). |
| 478 |
# If we didn't have the primary key check, then pk__in (== place__in) would |
| 479 |
# give Place's opts as the target opts, but Restaurant isn't compatible |
| 480 |
# with that. This logic applies only to primary keys, as when doing __in=qs, |
| 481 |
# we are going to turn this into __in=qs.values('pk') later on. |
| 482 |
return check(target_opts) or ( |
| 483 |
getattr(field, "primary_key", False) and check(field.model._meta) |
| 484 |
) |
| 485 |
|
| 486 |
|
| 487 |
class FilteredRelation: |
| 488 |
"""Specify custom filtering in the ON clause of SQL joins.""" |
| 489 |
|
| 490 |
def __init__(self, relation_name, *, condition=Q()): |
| 491 |
if not relation_name: |
| 492 |
raise ValueError("relation_name cannot be empty.") |
| 493 |
self.relation_name = relation_name |
| 494 |
self.alias = None |
| 495 |
if not isinstance(condition, Q): |
| 496 |
raise ValueError("condition argument must be a Q() instance.") |
| 497 |
# .condition and .resolved_condition have to be stored independently |
| 498 |
# as the former must remain unchanged for Join.__eq__ to remain stable |
| 499 |
# and reusable even once their .filtered_relation are resolved. |
| 500 |
self.condition = condition |
| 501 |
self.resolved_condition = None |
| 502 |
|
| 503 |
def __eq__(self, other): |
| 504 |
if not isinstance(other, self.__class__): |
| 505 |
return NotImplemented |
| 506 |
return ( |
| 507 |
self.relation_name == other.relation_name |
| 508 |
and self.alias == other.alias |
| 509 |
and self.condition == other.condition |
| 510 |
) |
| 511 |
|
| 512 |
def clone(self): |
| 513 |
clone = FilteredRelation(self.relation_name, condition=self.condition) |
| 514 |
clone.alias = self.alias |
| 515 |
if (resolved_condition := self.resolved_condition) is not None: |
| 516 |
clone.resolved_condition = resolved_condition.clone() |
| 517 |
return clone |
| 518 |
|
| 519 |
def relabeled_clone(self, change_map): |
| 520 |
clone = self.clone() |
| 521 |
if resolved_condition := clone.resolved_condition: |
| 522 |
clone.resolved_condition = resolved_condition.relabeled_clone(change_map) |
| 523 |
return clone |
| 524 |
|
| 525 |
def resolve_expression(self, query, reuse, *args, **kwargs): |
| 526 |
clone = self.clone() |
| 527 |
clone.resolved_condition = query.build_filter( |
| 528 |
self.condition, |
| 529 |
can_reuse=reuse, |
| 530 |
allow_joins=True, |
| 531 |
split_subq=False, |
| 532 |
update_join_types=False, |
| 533 |
)[0] |
| 534 |
return clone |
| 535 |
|
| 536 |
def as_sql(self, compiler, connection): |
| 537 |
return compiler.compile(self.resolved_condition) |