Debian Codegraph @

django/db/models/sql/query.py

raw
1 """
2 Create SQL statements for QuerySets.
3
4 The code in here encapsulates all of the SQL construction so that QuerySets
5 themselves do not have to (and could be backed by things other than SQL
6 databases). The abstraction barrier only works one way: this module has to know
7 all about the internals of models in order to get the information it needs.
8 """
9
10 import copy
11 import difflib
12 import functools
13 import sys
14 from collections import Counter, namedtuple
15 from collections.abc import Iterator, Mapping
16 from itertools import chain, count, product
17 from string import ascii_uppercase
18
19 from django.core.exceptions import FieldDoesNotExist, FieldError
20 from django.db import DEFAULT_DB_ALIAS, NotSupportedError, connections
21 from django.db.models.aggregates import Count
22 from django.db.models.constants import LOOKUP_SEP
23 from django.db.models.expressions import (
24 BaseExpression,
25 Col,
26 ColPairs,
27 Exists,
28 F,
29 OuterRef,
30 RawSQL,
31 Ref,
32 ResolvedOuterRef,
33 Value,
34 )
35 from django.db.models.fields import Field
36 from django.db.models.lookups import Lookup
37 from django.db.models.query_utils import (
38 Q,
39 check_rel_lookup_compatibility,
40 refs_expression,
41 )
42 from django.db.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE
43 from django.db.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin
44 from django.db.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode
45 from django.utils.functional import cached_property
46 from django.utils.regex_helper import _lazy_re_compile
47 from django.utils.tree import Node
48
49 __all__ = ["Query", "RawQuery"]
50
51 # Quotation marks ('"`[]), whitespace characters, control characters,
52 # semicolons, hashes, or inline SQL comments are forbidden in column aliases.
53 FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(
54 r"['`\"\]\[;\s\x00-\x1F\x7F-\x9F]|#|--|/\*|\*/"
55 )
56
57 # Inspired from
58 # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
59 EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w-]+")
60
61
62 def get_field_names_from_opts(opts):
63 if opts is None:
64 return set()
65 return set(
66 chain.from_iterable(
67 (f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields()
68 )
69 )
70
71
72 def get_paths_from_expression(expr):
73 if isinstance(expr, F):
74 yield expr.name
75 elif hasattr(expr, "flatten"):
76 for child in expr.flatten():
77 if isinstance(child, F):
78 yield child.name
79 elif isinstance(child, Q):
80 yield from get_children_from_q(child)
81
82
83 def get_children_from_q(q):
84 for child in q.children:
85 if isinstance(child, Node):
86 yield from get_children_from_q(child)
87 elif isinstance(child, tuple):
88 lhs, rhs = child
89 yield lhs
90 if hasattr(rhs, "resolve_expression"):
91 yield from get_paths_from_expression(rhs)
92 elif hasattr(child, "resolve_expression"):
93 yield from get_paths_from_expression(child)
94
95
96 def get_child_with_renamed_prefix(prefix, replacement, child):
97 from django.db.models.query import QuerySet
98
99 if isinstance(child, Node):
100 return rename_prefix_from_q(prefix, replacement, child)
101 if isinstance(child, tuple):
102 lhs, rhs = child
103 if lhs.startswith(prefix + LOOKUP_SEP):
104 lhs = lhs.replace(prefix, replacement, 1)
105 if not isinstance(rhs, F) and hasattr(rhs, "resolve_expression"):
106 rhs = get_child_with_renamed_prefix(prefix, replacement, rhs)
107 return lhs, rhs
108
109 if isinstance(child, F):
110 child = child.copy()
111 if child.name.startswith(prefix + LOOKUP_SEP):
112 child.name = child.name.replace(prefix, replacement, 1)
113 elif isinstance(child, QuerySet):
114 # QuerySet may contain OuterRef() references which cannot work properly
115 # without repointing to the filtered annotation and will spawn a
116 # different JOIN. Always raise ValueError instead of providing partial
117 # support in other cases.
118 raise ValueError(
119 "Passing a QuerySet within a FilteredRelation is not supported."
120 )
121 elif hasattr(child, "resolve_expression"):
122 child = child.copy()
123 child.set_source_expressions(
124 [
125 get_child_with_renamed_prefix(prefix, replacement, grand_child)
126 for grand_child in child.get_source_expressions()
127 ]
128 )
129 return child
130
131
132 def rename_prefix_from_q(prefix, replacement, q):
133 return Q.create(
134 [get_child_with_renamed_prefix(prefix, replacement, c) for c in q.children],
135 q.connector,
136 q.negated,
137 )
138
139
140 JoinInfo = namedtuple(
141 "JoinInfo",
142 ("final_field", "targets", "opts", "joins", "path", "transform_function"),
143 )
144
145
146 class RawQuery:
147 """A single raw SQL query."""
148
149 def __init__(self, sql, using, params=()):
150 self.params = params
151 self.sql = sql
152 self.using = using
153 self.cursor = None
154
155 # Mirror some properties of a normal query so that
156 # the compiler can be used to process results.
157 self.low_mark, self.high_mark = 0, None # Used for offset/limit
158 self.extra_select = {}
159 self.annotation_select = {}
160
161 def chain(self, using):
162 return self.clone(using)
163
164 def clone(self, using):
165 return RawQuery(self.sql, using, params=self.params)
166
167 def get_columns(self):
168 if self.cursor is None:
169 self._execute_query()
170 converter = connections[self.using].introspection.identifier_converter
171 return [converter(column_meta[0]) for column_meta in self.cursor.description]
172
173 def __iter__(self):
174 # Always execute a new query for a new iterator.
175 # This could be optimized with a cache at the expense of RAM.
176 self._execute_query()
177 if not connections[self.using].features.can_use_chunked_reads:
178 # If the database can't use chunked reads we need to make sure we
179 # evaluate the entire query up front.
180 result = list(self.cursor)
181 else:
182 result = self.cursor
183 return iter(result)
184
185 def __repr__(self):
186 return "<%s: %s>" % (self.__class__.__name__, self)
187
188 @property
189 def params_type(self):
190 if self.params is None:
191 return None
192 return dict if isinstance(self.params, Mapping) else tuple
193
194 def __str__(self):
195 if self.params_type is None:
196 return self.sql
197 return self.sql % self.params_type(self.params)
198
199 def _execute_query(self):
200 connection = connections[self.using]
201
202 # Adapt parameters to the database, as much as possible considering
203 # that the target type isn't known. See #17755.
204 params_type = self.params_type
205 adapter = connection.ops.adapt_unknown_value
206 if params_type is tuple:
207 params = tuple(adapter(val) for val in self.params)
208 elif params_type is dict:
209 params = {key: adapter(val) for key, val in self.params.items()}
210 elif params_type is None:
211 params = None
212 else:
213 raise RuntimeError("Unexpected params type: %s" % params_type)
214
215 self.cursor = connection.cursor()
216 self.cursor.execute(self.sql, params)
217
218
219 ExplainInfo = namedtuple("ExplainInfo", ("format", "options"))
220
221
222 class Query(BaseExpression):
223 """A single SQL query."""
224
225 alias_prefix = "T"
226 empty_result_set_value = None
227 subq_aliases = frozenset([alias_prefix])
228
229 compiler = "SQLCompiler"
230
231 base_table_class = BaseTable
232 join_class = Join
233
234 default_cols = True
235 default_ordering = True
236 standard_ordering = True
237
238 filter_is_sticky = False
239 subquery = False
240
241 # SQL-related attributes.
242 # Select and related select clauses are expressions to use in the SELECT
243 # clause of the query. The select is used for cases where we want to set up
244 # the select clause to contain other than default fields (values(),
245 # subqueries...). Note that annotations go to annotations dictionary.
246 select = ()
247 # The group_by attribute can have one of the following forms:
248 # - None: no group by at all in the query
249 # - A tuple of expressions: group by (at least) those expressions.
250 # String refs are also allowed for now.
251 # - True: group by all select fields of the model
252 # See compiler.get_group_by() for details.
253 group_by = None
254 order_by = ()
255 low_mark = 0 # Used for offset/limit.
256 high_mark = None # Used for offset/limit.
257 distinct = False
258 distinct_fields = ()
259 select_for_update = False
260 select_for_update_nowait = False
261 select_for_update_skip_locked = False
262 select_for_update_of = ()
263 select_for_no_key_update = False
264 select_related = False
265 # Arbitrary limit for select_related to prevents infinite recursion.
266 max_depth = 5
267 # Holds the selects defined by a call to values() or values_list()
268 # excluding annotation_select and extra_select.
269 values_select = ()
270 selected = None
271
272 # SQL annotation-related attributes.
273 annotation_select_mask = None
274 _annotation_select_cache = None
275
276 # Set combination attributes.
277 combinator = None
278 combinator_all = False
279 combined_queries = ()
280
281 # These are for extensions. The contents are more or less appended verbatim
282 # to the appropriate clause.
283 extra_select_mask = None
284 _extra_select_cache = None
285
286 extra_tables = ()
287 extra_order_by = ()
288
289 # A tuple that is a set of model field names and either True, if these are
290 # the fields to defer, or False if these are the only fields to load.
291 deferred_loading = (frozenset(), True)
292
293 explain_info = None
294
295 def __init__(self, model, alias_cols=True):
296 self.model = model
297 self.alias_refcount = {}
298 # alias_map is the most important data structure regarding joins.
299 # It's used for recording which joins exist in the query and what
300 # types they are. The key is the alias of the joined table (possibly
301 # the table name) and the value is a Join-like object (see
302 # sql.datastructures.Join for more information).
303 self.alias_map = {}
304 # Whether to provide alias to columns during reference resolving.
305 self.alias_cols = alias_cols
306 # Sometimes the query contains references to aliases in outer queries (as
307 # a result of split_exclude). Correct alias quoting needs to know these
308 # aliases too.
309 # Map external tables to whether they are aliased.
310 self.external_aliases = {}
311 self.table_map = {} # Maps table names to list of aliases.
312 self.used_aliases = set()
313
314 self.where = WhereNode()
315 # Maps alias -> Annotation Expression.
316 self.annotations = {}
317 # These are for extensions. The contents are more or less appended
318 # verbatim to the appropriate clause.
319 self.extra = {} # Maps col_alias -> (col_sql, params).
320
321 self._filtered_relations = {}
322
323 @property
324 def output_field(self):
325 if len(self.select) == 1:
326 select = self.select[0]
327 return getattr(select, "target", None) or select.field
328 elif len(self.annotation_select) == 1:
329 return next(iter(self.annotation_select.values())).output_field
330
331 @cached_property
332 def base_table(self):
333 for alias in self.alias_map:
334 return alias
335
336 def __str__(self):
337 """
338 Return the query as a string of SQL with the parameter values
339 substituted in (use sql_with_params() to see the unsubstituted string).
340
341 Parameter values won't necessarily be quoted correctly, since that is
342 done by the database interface at execution time.
343 """
344 sql, params = self.sql_with_params()
345 return sql % params
346
347 def sql_with_params(self):
348 """
349 Return the query as an SQL string and the parameters that will be
350 substituted into the query.
351 """
352 return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()
353
354 def __deepcopy__(self, memo):
355 """Limit the amount of work when a Query is deepcopied."""
356 result = self.clone()
357 memo[id(self)] = result
358 return result
359
360 def get_compiler(self, using=None, connection=None, elide_empty=True):
361 if using is None and connection is None:
362 raise ValueError("Need either using or connection")
363 if using:
364 connection = connections[using]
365 return connection.ops.compiler(self.compiler)(
366 self, connection, using, elide_empty
367 )
368
369 def get_meta(self):
370 """
371 Return the Options instance (the model._meta) from which to start
372 processing. Normally, this is self.model._meta, but it can be changed
373 by subclasses.
374 """
375 if self.model:
376 return self.model._meta
377
378 def clone(self):
379 """
380 Return a copy of the current Query. A lightweight alternative to
381 deepcopy().
382 """
383 obj = Empty()
384 obj.__class__ = self.__class__
385 # Copy references to everything.
386 obj.__dict__ = self.__dict__.copy()
387 # Clone attributes that can't use shallow copy.
388 obj.alias_refcount = self.alias_refcount.copy()
389 obj.alias_map = self.alias_map.copy()
390 obj.external_aliases = self.external_aliases.copy()
391 obj.table_map = self.table_map.copy()
392 obj.where = self.where.clone()
393 obj.annotations = self.annotations.copy()
394 if self.annotation_select_mask is not None:
395 obj.annotation_select_mask = self.annotation_select_mask.copy()
396 if self.combined_queries:
397 obj.combined_queries = tuple(
398 [query.clone() for query in self.combined_queries]
399 )
400 # _annotation_select_cache cannot be copied, as doing so breaks the
401 # (necessary) state in which both annotations and
402 # _annotation_select_cache point to the same underlying objects.
403 # It will get re-populated in the cloned queryset the next time it's
404 # used.
405 obj._annotation_select_cache = None
406 obj.extra = self.extra.copy()
407 if self.extra_select_mask is not None:
408 obj.extra_select_mask = self.extra_select_mask.copy()
409 if self._extra_select_cache is not None:
410 obj._extra_select_cache = self._extra_select_cache.copy()
411 if self.select_related is not False:
412 # Use deepcopy because select_related stores fields in nested
413 # dicts.
414 obj.select_related = copy.deepcopy(obj.select_related)
415 if "subq_aliases" in self.__dict__:
416 obj.subq_aliases = self.subq_aliases.copy()
417 obj.used_aliases = self.used_aliases.copy()
418 obj._filtered_relations = self._filtered_relations.copy()
419 # Clear the cached_property, if it exists.
420 obj.__dict__.pop("base_table", None)
421 return obj
422
423 def chain(self, klass=None):
424 """
425 Return a copy of the current Query that's ready for another operation.
426 The klass argument changes the type of the Query, e.g. UpdateQuery.
427 """
428 obj = self.clone()
429 if klass and obj.__class__ != klass:
430 obj.__class__ = klass
431 if not obj.filter_is_sticky:
432 obj.used_aliases = set()
433 obj.filter_is_sticky = False
434 if hasattr(obj, "_setup_query"):
435 obj._setup_query()
436 return obj
437
438 def relabeled_clone(self, change_map):
439 clone = self.clone()
440 clone.change_aliases(change_map)
441 return clone
442
443 def _get_col(self, target, field, alias):
444 if not self.alias_cols:
445 alias = None
446 return target.get_col(alias, field)
447
448 def get_aggregation(self, using, aggregate_exprs):
449 """
450 Return the dictionary with the values of the existing aggregations.
451 """
452 if not aggregate_exprs:
453 return {}
454 # Store annotation mask prior to temporarily adding aggregations for
455 # resolving purpose to facilitate their subsequent removal.
456 refs_subquery = False
457 refs_window = False
458 replacements = {}
459 annotation_select_mask = self.annotation_select_mask
460 for alias, aggregate_expr in aggregate_exprs.items():
461 self.check_alias(alias)
462 aggregate = aggregate_expr.resolve_expression(
463 self, allow_joins=True, reuse=None, summarize=True
464 )
465 if not aggregate.contains_aggregate:
466 raise TypeError("%s is not an aggregate expression" % alias)
467 # Temporarily add aggregate to annotations to allow remaining
468 # members of `aggregates` to resolve against each others.
469 self.append_annotation_mask([alias])
470 aggregate_refs = aggregate.get_refs()
471 refs_subquery |= any(
472 getattr(self.annotations[ref], "contains_subquery", False)
473 for ref in aggregate_refs
474 )
475 refs_window |= any(
476 getattr(self.annotations[ref], "contains_over_clause", True)
477 for ref in aggregate_refs
478 )
479 aggregate = aggregate.replace_expressions(replacements)
480 self.annotations[alias] = aggregate
481 replacements[Ref(alias, aggregate)] = aggregate
482 # Stash resolved aggregates now that they have been allowed to resolve
483 # against each other.
484 aggregates = {alias: self.annotations.pop(alias) for alias in aggregate_exprs}
485 self.set_annotation_mask(annotation_select_mask)
486 # Existing usage of aggregation can be determined by the presence of
487 # selected aggregates but also by filters against aliased aggregates.
488 _, having, qualify = self.where.split_having_qualify()
489 has_existing_aggregation = (
490 any(
491 getattr(annotation, "contains_aggregate", True)
492 for annotation in self.annotations.values()
493 )
494 or having
495 )
496 set_returning_annotations = {
497 alias
498 for alias, annotation in self.annotation_select.items()
499 if getattr(annotation, "set_returning", False)
500 }
501 # Decide if we need to use a subquery.
502 #
503 # Existing aggregations would cause incorrect results as
504 # get_aggregation() must produce just one result and thus must not use
505 # GROUP BY.
506 #
507 # If the query has limit or distinct, or uses set operations, then
508 # those operations must be done in a subquery so that the query
509 # aggregates on the limit and/or distinct results instead of applying
510 # the distinct and limit after the aggregation.
511 if (
512 isinstance(self.group_by, tuple)
513 or self.is_sliced
514 or has_existing_aggregation
515 or refs_subquery
516 or refs_window
517 or qualify
518 or self.distinct
519 or self.combinator
520 or set_returning_annotations
521 ):
522 from django.db.models.sql.subqueries import AggregateQuery
523
524 inner_query = self.clone()
525 inner_query.subquery = True
526 outer_query = AggregateQuery(self.model, inner_query)
527 inner_query.select_for_update = False
528 inner_query.select_related = False
529 inner_query.set_annotation_mask(self.annotation_select)
530 # Queries with distinct_fields need ordering and when a limit is
531 # applied we must take the slice from the ordered query. Otherwise
532 # no need for ordering.
533 inner_query.clear_ordering(force=False)
534 if not inner_query.distinct:
535 # If the inner query uses default select and it has some
536 # aggregate annotations, then we must make sure the inner
537 # query is grouped by the main model's primary key. However,
538 # clearing the select clause can alter results if distinct is
539 # used.
540 if inner_query.default_cols and has_existing_aggregation:
541 inner_query.group_by = (
542 self.model._meta.pk.get_col(inner_query.get_initial_alias()),
543 )
544 inner_query.default_cols = False
545 if not qualify and not self.combinator:
546 # Mask existing annotations that are not referenced by
547 # aggregates to be pushed to the outer query unless
548 # filtering against window functions or if the query is
549 # combined as both would require complex realiasing logic.
550 annotation_mask = set()
551 if isinstance(self.group_by, tuple):
552 for expr in self.group_by:
553 annotation_mask |= expr.get_refs()
554 for aggregate in aggregates.values():
555 annotation_mask |= aggregate.get_refs()
556 # Avoid eliding expressions that might have an incidence on
557 # the implicit grouping logic.
558 for annotation_alias, annotation in self.annotation_select.items():
559 if annotation.get_group_by_cols():
560 annotation_mask.add(annotation_alias)
561 inner_query.set_annotation_mask(annotation_mask)
562 # Annotations that possibly return multiple rows cannot
563 # be masked as they might have an incidence on the query.
564 annotation_mask |= set_returning_annotations
565
566 # Add aggregates to the outer AggregateQuery. This requires making
567 # sure all columns referenced by the aggregates are selected in the
568 # inner query. It is achieved by retrieving all column references
569 # by the aggregates, explicitly selecting them in the inner query,
570 # and making sure the aggregates are repointed to them.
571 col_refs = {}
572 for alias, aggregate in aggregates.items():
573 replacements = {}
574 for col in self._gen_cols([aggregate], resolve_refs=False):
575 if not (col_ref := col_refs.get(col)):
576 index = len(col_refs) + 1
577 col_alias = f"__col{index}"
578 col_ref = Ref(col_alias, col)
579 col_refs[col] = col_ref
580 inner_query.add_annotation(col, col_alias)
581 replacements[col] = col_ref
582 outer_query.annotations[alias] = aggregate.replace_expressions(
583 replacements
584 )
585 if (
586 inner_query.select == ()
587 and not inner_query.default_cols
588 and not inner_query.annotation_select_mask
589 ):
590 # In case of Model.objects[0:3].count(), there would be no
591 # field selected in the inner query, yet we must use a subquery.
592 # So, make sure at least one field is selected.
593 inner_query.select = (
594 self.model._meta.pk.get_col(inner_query.get_initial_alias()),
595 )
596 else:
597 outer_query = self
598 self.select = ()
599 self.selected = None
600 self.default_cols = False
601 self.extra = {}
602 if self.annotations:
603 # Inline reference to existing annotations and mask them as
604 # they are unnecessary given only the summarized aggregations
605 # are requested.
606 replacements = {
607 Ref(alias, annotation): annotation
608 for alias, annotation in self.annotations.items()
609 }
610 self.annotations = {
611 alias: aggregate.replace_expressions(replacements)
612 for alias, aggregate in aggregates.items()
613 }
614 else:
615 self.annotations = aggregates
616 self.set_annotation_mask(aggregates)
617
618 empty_set_result = [
619 expression.empty_result_set_value
620 for expression in outer_query.annotation_select.values()
621 ]
622 elide_empty = not any(result is NotImplemented for result in empty_set_result)
623 outer_query.clear_ordering(force=True)
624 outer_query.clear_limits()
625 outer_query.select_for_update = False
626 outer_query.select_related = False
627 compiler = outer_query.get_compiler(using, elide_empty=elide_empty)
628 result = compiler.execute_sql(SINGLE)
629 if result is None:
630 result = empty_set_result
631 else:
632 cols = outer_query.annotation_select.values()
633 converters = compiler.get_converters(cols)
634 rows = compiler.apply_converters((result,), converters)
635 if compiler.has_composite_fields(cols):
636 rows = compiler.composite_fields_to_tuples(rows, cols)
637 result = next(rows)
638
639 return dict(zip(outer_query.annotation_select, result))
640
641 def get_count(self, using):
642 """
643 Perform a COUNT() query using the current filter constraints.
644 """
645 obj = self.clone()
646 return obj.get_aggregation(using, {"__count": Count("*")})["__count"]
647
648 def has_filters(self):
649 return self.where
650
651 def exists(self, limit=True):
652 q = self.clone()
653 if not (q.distinct and q.is_sliced):
654 if q.group_by is True:
655 q.add_fields(
656 (f.attname for f in self.model._meta.concrete_fields), False
657 )
658 # Disable GROUP BY aliases to avoid orphaning references to the
659 # SELECT clause which is about to be cleared.
660 q.set_group_by(allow_aliases=False)
661 q.clear_select_clause()
662 if q.combined_queries and q.combinator == "union":
663 q.combined_queries = tuple(
664 combined_query.exists(limit=False)
665 for combined_query in q.combined_queries
666 )
667 q.clear_ordering(force=True)
668 if limit is True:
669 q.set_limits(high=1)
670 q.add_annotation(Value(1), "a")
671 return q
672
673 def has_results(self, using):
674 q = self.exists()
675 compiler = q.get_compiler(using=using)
676 return compiler.has_results()
677
678 def explain(self, using, format=None, **options):
679 q = self.clone()
680 for option_name in options:
681 if (
682 not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name)
683 or "--" in option_name
684 ):
685 raise ValueError(f"Invalid option name: {option_name!r}.")
686 q.explain_info = ExplainInfo(format, options)
687 compiler = q.get_compiler(using=using)
688 return "\n".join(compiler.explain_query())
689
690 def combine(self, rhs, connector):
691 """
692 Merge the 'rhs' query into the current one (with any 'rhs' effects
693 being applied *after* (that is, "to the right of") anything in the
694 current query. 'rhs' is not modified during a call to this function.
695
696 The 'connector' parameter describes how to connect filters from the
697 'rhs' query.
698 """
699 if self.model != rhs.model:
700 raise TypeError("Cannot combine queries on two different base models.")
701 if self.is_sliced:
702 raise TypeError("Cannot combine queries once a slice has been taken.")
703 if self.distinct != rhs.distinct:
704 raise TypeError("Cannot combine a unique query with a non-unique query.")
705 if self.distinct_fields != rhs.distinct_fields:
706 raise TypeError("Cannot combine queries with different distinct fields.")
707
708 # If lhs and rhs shares the same alias prefix, it is possible to have
709 # conflicting alias changes like T4 -> T5, T5 -> T6, which might end up
710 # as T4 -> T6 while combining two querysets. To prevent this, change an
711 # alias prefix of the rhs and update current aliases accordingly,
712 # except if the alias is the base table since it must be present in the
713 # query on both sides.
714 initial_alias = self.get_initial_alias()
715 rhs = rhs.clone()
716 rhs.bump_prefix(self, exclude={initial_alias})
717
718 # Work out how to relabel the rhs aliases, if necessary.
719 change_map = {}
720 conjunction = connector == AND
721
722 # Determine which existing joins can be reused. When combining the
723 # query with AND we must recreate all joins for m2m filters. When
724 # combining with OR we can reuse joins. The reason is that in AND
725 # case a single row can't fulfill a condition like:
726 # revrel__col=1 & revrel__col=2
727 # But, there might be two different related rows matching this
728 # condition. In OR case a single True is enough, so single row is
729 # enough, too.
730 #
731 # Note that we will be creating duplicate joins for non-m2m joins in
732 # the AND case. The results will be correct but this creates too many
733 # joins. This is something that could be fixed later on.
734 reuse = set() if conjunction else set(self.alias_map)
735 joinpromoter = JoinPromoter(connector, 2, False)
736 joinpromoter.add_votes(
737 j for j in self.alias_map if self.alias_map[j].join_type == INNER
738 )
739 rhs_votes = set()
740 # Now, add the joins from rhs query into the new query (skipping base
741 # table).
742 rhs_tables = list(rhs.alias_map)[1:]
743 for alias in rhs_tables:
744 join = rhs.alias_map[alias]
745 # If the left side of the join was already relabeled, use the
746 # updated alias.
747 join = join.relabeled_clone(change_map)
748 new_alias = self.join(join, reuse=reuse)
749 if join.join_type == INNER:
750 rhs_votes.add(new_alias)
751 # We can't reuse the same join again in the query. If we have two
752 # distinct joins for the same connection in rhs query, then the
753 # combined query must have two joins, too.
754 reuse.discard(new_alias)
755 if alias != new_alias:
756 change_map[alias] = new_alias
757 if not rhs.alias_refcount[alias]:
758 # The alias was unused in the rhs query. Unref it so that it
759 # will be unused in the new query, too. We have to add and
760 # unref the alias so that join promotion has information of
761 # the join type for the unused alias.
762 self.unref_alias(new_alias)
763 joinpromoter.add_votes(rhs_votes)
764 joinpromoter.update_join_types(self)
765
766 # Combine subqueries aliases to ensure aliases relabelling properly
767 # handle subqueries when combining where and select clauses.
768 self.subq_aliases |= rhs.subq_aliases
769
770 # Now relabel a copy of the rhs where-clause and add it to the current
771 # one.
772 w = rhs.where.clone()
773 w.relabel_aliases(change_map)
774 self.where.add(w, connector)
775
776 # Selection columns and extra extensions are those provided by 'rhs'.
777 if rhs.select:
778 self.set_select([col.relabeled_clone(change_map) for col in rhs.select])
779 else:
780 self.select = ()
781
782 if connector == OR:
783 # It would be nice to be able to handle this, but the queries don't
784 # really make sense (or return consistent value sets). Not worth
785 # the extra complexity when you can write a real query instead.
786 if self.extra and rhs.extra:
787 raise ValueError(
788 "When merging querysets using 'or', you cannot have "
789 "extra(select=...) on both sides."
790 )
791 self.extra.update(rhs.extra)
792 extra_select_mask = set()
793 if self.extra_select_mask is not None:
794 extra_select_mask.update(self.extra_select_mask)
795 if rhs.extra_select_mask is not None:
796 extra_select_mask.update(rhs.extra_select_mask)
797 if extra_select_mask:
798 self.set_extra_mask(extra_select_mask)
799 self.extra_tables += rhs.extra_tables
800
801 # Ordering uses the 'rhs' ordering, unless it has none, in which case
802 # the current ordering is used.
803 self.order_by = rhs.order_by or self.order_by
804 self.extra_order_by = rhs.extra_order_by or self.extra_order_by
805
806 def _get_defer_select_mask(self, opts, mask, select_mask=None):
807 if select_mask is None:
808 select_mask = {}
809 select_mask[opts.pk] = {}
810 # All concrete fields and related objects that are not part of the
811 # defer mask must be included. If a relational field is encountered it
812 # gets added to the mask for it be considered if `select_related` and
813 # the cycle continues by recursively calling this function.
814 for field in opts.concrete_fields + opts.related_objects:
815 field_mask = mask.pop(field.name, None)
816 field_att_mask = None
817 if field_attname := getattr(field, "attname", None):
818 field_att_mask = mask.pop(field_attname, None)
819 if field_mask is None and field_att_mask is None:
820 select_mask.setdefault(field, {})
821 elif field_mask:
822 if not field.is_relation:
823 raise FieldError(next(iter(field_mask)))
824 # Virtual fields such as many-to-many and generic foreign keys
825 # cannot be effectively deferred. Historically, they were
826 # allowed to be passed to QuerySet.defer(). Ignore such field
827 # references until a layer of validation at mask alteration
828 # time is eventually implemented.
829 if field.many_to_many:
830 continue
831 field_select_mask = select_mask.setdefault(field, {})
832 related_model = field.related_model._meta.concrete_model
833 self._get_defer_select_mask(
834 related_model._meta, field_mask, field_select_mask
835 )
836 # Remaining defer entries must be references to filtered relations
837 # otherwise they are surfaced as missing field errors.
838 for field_name, field_mask in mask.items():
839 if filtered_relation := self._filtered_relations.get(field_name):
840 relation = opts.get_field(filtered_relation.relation_name)
841 field_select_mask = select_mask.setdefault((field_name, relation), {})
842 related_model = relation.related_model._meta.concrete_model
843 self._get_defer_select_mask(
844 related_model._meta, field_mask, field_select_mask
845 )
846 else:
847 opts.get_field(field_name)
848 return select_mask
849
850 def _get_only_select_mask(self, opts, mask, select_mask=None):
851 if select_mask is None:
852 select_mask = {}
853 select_mask[opts.pk] = {}
854 # Only include fields mentioned in the mask.
855 for field_name, field_mask in mask.items():
856 field = opts.get_field(field_name)
857 field_select_mask = select_mask.setdefault(field, {})
858 if field_mask:
859 if not field.is_relation:
860 raise FieldError(next(iter(field_mask)))
861 related_model = field.remote_field.model._meta.concrete_model
862 self._get_only_select_mask(
863 related_model._meta, field_mask, field_select_mask
864 )
865 return select_mask
866
867 def get_select_mask(self):
868 """
869 Convert the self.deferred_loading data structure to an alternate data
870 structure, describing the field that *will* be loaded. This is used to
871 compute the columns to select from the database and also by the
872 QuerySet class to work out which fields are being initialized on each
873 model. Models that have all their fields included aren't mentioned in
874 the result, only those that have field restrictions in place.
875 """
876 field_names, defer = self.deferred_loading
877 if not field_names:
878 return {}
879 mask = {}
880 for field_name in field_names:
881 part_mask = mask
882 for part in field_name.split(LOOKUP_SEP):
883 part_mask = part_mask.setdefault(part, {})
884 opts = self.get_meta()
885 if defer:
886 return self._get_defer_select_mask(opts, mask)
887 return self._get_only_select_mask(opts, mask)
888
889 def table_alias(self, table_name, create=False, filtered_relation=None):
890 """
891 Return a table alias for the given table_name and whether this is a
892 new alias or not.
893
894 If 'create' is true, a new alias is always created. Otherwise, the
895 most recently created alias for the table (if one exists) is reused.
896 """
897 alias_list = self.table_map.get(table_name)
898 if not create and alias_list:
899 alias = alias_list[0]
900 self.alias_refcount[alias] += 1
901 return alias, False
902
903 # Create a new alias for this table.
904 if alias_list:
905 alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1)
906 alias_list.append(alias)
907 else:
908 # The first occurrence of a table uses the table name directly.
909 alias = (
910 filtered_relation.alias if filtered_relation is not None else table_name
911 )
912 self.table_map[table_name] = [alias]
913 self.alias_refcount[alias] = 1
914 return alias, True
915
916 def ref_alias(self, alias):
917 """Increases the reference count for this alias."""
918 self.alias_refcount[alias] += 1
919
920 def unref_alias(self, alias, amount=1):
921 """Decreases the reference count for this alias."""
922 self.alias_refcount[alias] -= amount
923
924 def promote_joins(self, aliases):
925 """
926 Promote recursively the join type of given aliases and its children to
927 an outer join. If 'unconditional' is False, only promote the join if
928 it is nullable or the parent join is an outer join.
929
930 The children promotion is done to avoid join chains that contain a LOUTER
931 b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted,
932 then we must also promote b->c automatically, or otherwise the promotion
933 of a->b doesn't actually change anything in the query results.
934 """
935 aliases = list(aliases)
936 while aliases:
937 alias = aliases.pop(0)
938 if self.alias_map[alias].join_type is None:
939 # This is the base table (first FROM entry) - this table
940 # isn't really joined at all in the query, so we should not
941 # alter its join type.
942 continue
943 # Only the first alias (skipped above) should have None join_type
944 assert self.alias_map[alias].join_type is not None
945 parent_alias = self.alias_map[alias].parent_alias
946 parent_louter = (
947 parent_alias and self.alias_map[parent_alias].join_type == LOUTER
948 )
949 already_louter = self.alias_map[alias].join_type == LOUTER
950 if (self.alias_map[alias].nullable or parent_louter) and not already_louter:
951 self.alias_map[alias] = self.alias_map[alias].promote()
952 # Join type of 'alias' changed, so re-examine all aliases that
953 # refer to this one.
954 aliases.extend(
955 join
956 for join in self.alias_map
957 if self.alias_map[join].parent_alias == alias
958 and join not in aliases
959 )
960
961 def demote_joins(self, aliases):
962 """
963 Change join type from LOUTER to INNER for all joins in aliases.
964
965 Similarly to promote_joins(), this method must ensure no join chains
966 containing first an outer, then an inner join are generated. If we
967 are demoting b->c join in chain a LOUTER b LOUTER c then we must
968 demote a->b automatically, or otherwise the demotion of b->c doesn't
969 actually change anything in the query results. .
970 """
971 aliases = list(aliases)
972 while aliases:
973 alias = aliases.pop(0)
974 if self.alias_map[alias].join_type == LOUTER:
975 self.alias_map[alias] = self.alias_map[alias].demote()
976 parent_alias = self.alias_map[alias].parent_alias
977 if self.alias_map[parent_alias].join_type == INNER:
978 aliases.append(parent_alias)
979
980 def reset_refcounts(self, to_counts):
981 """
982 Reset reference counts for aliases so that they match the value passed
983 in `to_counts`.
984 """
985 for alias, cur_refcount in self.alias_refcount.copy().items():
986 unref_amount = cur_refcount - to_counts.get(alias, 0)
987 self.unref_alias(alias, unref_amount)
988
989 def change_aliases(self, change_map):
990 """
991 Change the aliases in change_map (which maps old-alias -> new-alias),
992 relabelling any references to them in select columns and the where
993 clause.
994 """
995 if not change_map:
996 return self
997 # If keys and values of change_map were to intersect, an alias might be
998 # updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending
999 # on their order in change_map.
1000 assert set(change_map).isdisjoint(change_map.values())
1001
1002 # 1. Update references in "select" (normal columns plus aliases),
1003 # "group by" and "where".
1004 self.where.relabel_aliases(change_map)
1005 if isinstance(self.group_by, tuple):
1006 self.group_by = tuple(
1007 [col.relabeled_clone(change_map) for col in self.group_by]
1008 )
1009 self.select = tuple([col.relabeled_clone(change_map) for col in self.select])
1010 self.annotations = self.annotations and {
1011 key: col.relabeled_clone(change_map)
1012 for key, col in self.annotations.items()
1013 }
1014
1015 # 2. Rename the alias in the internal table/alias datastructures.
1016 for old_alias, new_alias in change_map.items():
1017 if old_alias not in self.alias_map:
1018 continue
1019 alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
1020 self.alias_map[new_alias] = alias_data
1021 self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
1022 del self.alias_refcount[old_alias]
1023 del self.alias_map[old_alias]
1024
1025 table_aliases = self.table_map[alias_data.table_name]
1026 for pos, alias in enumerate(table_aliases):
1027 if alias == old_alias:
1028 table_aliases[pos] = new_alias
1029 break
1030
1031 # 3. Rename the direct external aliases and the ones of combined
1032 # queries (union, intersection, difference).
1033 self.external_aliases = {
1034 # Table is aliased or it's being changed and thus is aliased.
1035 change_map.get(alias, alias): (aliased or alias in change_map)
1036 for alias, aliased in self.external_aliases.items()
1037 }
1038 for combined_query in self.combined_queries:
1039 external_change_map = {
1040 alias: aliased
1041 for alias, aliased in change_map.items()
1042 if alias in combined_query.external_aliases
1043 }
1044 combined_query.change_aliases(external_change_map)
1045
1046 def bump_prefix(self, other_query, exclude=None):
1047 """
1048 Change the alias prefix to the next letter in the alphabet in a way
1049 that the other query's aliases and this query's aliases will not
1050 conflict. Even tables that previously had no alias will get an alias
1051 after this call. To prevent changing aliases use the exclude parameter.
1052 """
1053
1054 def prefix_gen():
1055 """
1056 Generate a sequence of characters in alphabetical order:
1057 -> 'A', 'B', 'C', ...
1058
1059 When the alphabet is finished, the sequence will continue with the
1060 Cartesian product:
1061 -> 'AA', 'AB', 'AC', ...
1062 """
1063 alphabet = ascii_uppercase
1064 prefix = chr(ord(self.alias_prefix) + 1)
1065 yield prefix
1066 for n in count(1):
1067 seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet
1068 for s in product(seq, repeat=n):
1069 yield "".join(s)
1070 prefix = None
1071
1072 if self.alias_prefix != other_query.alias_prefix:
1073 # No clashes between self and outer query should be possible.
1074 return
1075
1076 # Explicitly avoid infinite loop. The constant divider is based on how
1077 # much depth recursive subquery references add to the stack. This value
1078 # might need to be adjusted when adding or removing function calls from
1079 # the code path in charge of performing these operations.
1080 local_recursion_limit = sys.getrecursionlimit() // 16
1081 for pos, prefix in enumerate(prefix_gen()):
1082 if prefix not in self.subq_aliases:
1083 self.alias_prefix = prefix
1084 break
1085 if pos > local_recursion_limit:
1086 raise RecursionError(
1087 "Maximum recursion depth exceeded: too many subqueries."
1088 )
1089 self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
1090 other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases)
1091 if exclude is None:
1092 exclude = {}
1093 self.change_aliases(
1094 {
1095 alias: "%s%d" % (self.alias_prefix, pos)
1096 for pos, alias in enumerate(self.alias_map)
1097 if alias not in exclude
1098 }
1099 )
1100
1101 def get_initial_alias(self):
1102 """
1103 Return the first alias for this query, after increasing its reference
1104 count.
1105 """
1106 if self.alias_map:
1107 alias = self.base_table
1108 self.ref_alias(alias)
1109 elif self.model:
1110 alias = self.join(self.base_table_class(self.get_meta().db_table, None))
1111 else:
1112 alias = None
1113 return alias
1114
1115 def count_active_tables(self):
1116 """
1117 Return the number of tables in this query with a non-zero reference
1118 count. After execution, the reference counts are zeroed, so tables
1119 added in compiler will not be seen by this method.
1120 """
1121 return len([1 for count in self.alias_refcount.values() if count])
1122
1123 def join(self, join, reuse=None):
1124 """
1125 Return an alias for the 'join', either reusing an existing alias for
1126 that join or creating a new one. 'join' is either a base_table_class or
1127 join_class.
1128
1129 The 'reuse' parameter can be either None which means all joins are
1130 reusable, or it can be a set containing the aliases that can be reused.
1131
1132 A join is always created as LOUTER if the lhs alias is LOUTER to make
1133 sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new
1134 joins are created as LOUTER if the join is nullable.
1135 """
1136 reuse_aliases = [
1137 a
1138 for a, j in self.alias_map.items()
1139 if (reuse is None or a in reuse) and j == join
1140 ]
1141 if reuse_aliases:
1142 if join.table_alias in reuse_aliases:
1143 reuse_alias = join.table_alias
1144 else:
1145 # Reuse the most recent alias of the joined table
1146 # (a many-to-many relation may be joined multiple times).
1147 reuse_alias = reuse_aliases[-1]
1148 self.ref_alias(reuse_alias)
1149 return reuse_alias
1150
1151 # No reuse is possible, so we need a new alias.
1152 alias, _ = self.table_alias(
1153 join.table_name, create=True, filtered_relation=join.filtered_relation
1154 )
1155 if join.join_type:
1156 if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
1157 join_type = LOUTER
1158 else:
1159 join_type = INNER
1160 join.join_type = join_type
1161 join.table_alias = alias
1162 self.alias_map[alias] = join
1163 if filtered_relation := join.filtered_relation:
1164 resolve_reuse = reuse
1165 if resolve_reuse is not None:
1166 resolve_reuse = set(reuse) | {alias}
1167 joins_len = len(self.alias_map)
1168 join.filtered_relation = filtered_relation.resolve_expression(
1169 self, reuse=resolve_reuse
1170 )
1171 # Some joins were during expression resolving, they must be present
1172 # before the one we just added.
1173 if joins_len < len(self.alias_map):
1174 self.alias_map[alias] = self.alias_map.pop(alias)
1175 return alias
1176
1177 def join_parent_model(self, opts, model, alias, seen):
1178 """
1179 Make sure the given 'model' is joined in the query. If 'model' isn't
1180 a parent of 'opts' or if it is None this method is a no-op.
1181
1182 The 'alias' is the root alias for starting the join, 'seen' is a dict
1183 of model -> alias of existing joins. It must also contain a mapping
1184 of None -> some alias. This will be returned in the no-op case.
1185 """
1186 if model in seen:
1187 return seen[model]
1188 chain = opts.get_base_chain(model)
1189 if not chain:
1190 return alias
1191 curr_opts = opts
1192 for int_model in chain:
1193 if int_model in seen:
1194 curr_opts = int_model._meta
1195 alias = seen[int_model]
1196 continue
1197 # Proxy model have elements in base chain
1198 # with no parents, assign the new options
1199 # object and skip to the next base in that
1200 # case
1201 if not curr_opts.parents[int_model]:
1202 curr_opts = int_model._meta
1203 continue
1204 link_field = curr_opts.get_ancestor_link(int_model)
1205 join_info = self.setup_joins([link_field.name], curr_opts, alias)
1206 curr_opts = int_model._meta
1207 alias = seen[int_model] = join_info.joins[-1]
1208 return alias or seen[None]
1209
1210 def check_alias(self, alias):
1211 if FORBIDDEN_ALIAS_PATTERN.search(alias):
1212 raise ValueError(
1213 "Column aliases cannot contain whitespace characters, hashes, "
1214 "control characters, quotation marks, semicolons, or SQL comments."
1215 )
1216
1217 def add_annotation(self, annotation, alias, select=True):
1218 """Add a single annotation expression to the Query."""
1219 self.check_alias(alias)
1220 annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)
1221 if select:
1222 self.append_annotation_mask([alias])
1223 else:
1224 self.set_annotation_mask(set(self.annotation_select).difference({alias}))
1225 self.annotations[alias] = annotation
1226 if select and self.selected:
1227 self.selected[alias] = alias
1228
1229 @property
1230 def _subquery_fields_len(self):
1231 if self.has_select_fields:
1232 return sum(
1233 len(self.model._meta.pk_fields) if field == "pk" else 1
1234 for field in self.selected
1235 )
1236 return len(self.model._meta.pk_fields)
1237
1238 def resolve_expression(self, query, *args, **kwargs):
1239 clone = self.clone()
1240 # Subqueries need to use a different set of aliases than the outer query.
1241 clone.bump_prefix(query)
1242 clone.subquery = True
1243 clone.where.resolve_expression(query, *args, **kwargs)
1244 # Resolve combined queries.
1245 if clone.combinator:
1246 clone.combined_queries = tuple(
1247 [
1248 combined_query.resolve_expression(query, *args, **kwargs)
1249 for combined_query in clone.combined_queries
1250 ]
1251 )
1252 for key, value in clone.annotations.items():
1253 resolved = value.resolve_expression(query, *args, **kwargs)
1254 if hasattr(resolved, "external_aliases"):
1255 resolved.external_aliases.update(clone.external_aliases)
1256 clone.annotations[key] = resolved
1257 # Outer query's aliases are considered external.
1258 for alias, table in query.alias_map.items():
1259 clone.external_aliases[alias] = (
1260 isinstance(table, Join)
1261 and table.join_field.related_model._meta.db_table != alias
1262 ) or (
1263 isinstance(table, BaseTable) and table.table_name != table.table_alias
1264 )
1265 return clone
1266
1267 def get_external_cols(self):
1268 exprs = chain(self.annotations.values(), self.where.children)
1269 return [
1270 col
1271 for col in self._gen_cols(exprs, include_external=True)
1272 if col.alias in self.external_aliases
1273 ]
1274
1275 def get_group_by_cols(self, wrapper=None):
1276 # If wrapper is referenced by an alias for an explicit GROUP BY through
1277 # values() a reference to this expression and not the self must be
1278 # returned to ensure external column references are not grouped against
1279 # as well.
1280 external_cols = self.get_external_cols()
1281 if any(col.possibly_multivalued for col in external_cols):
1282 return [wrapper or self]
1283 return external_cols
1284
1285 def as_sql(self, compiler, connection):
1286 # Some backends (e.g. Oracle) raise an error when a subquery contains
1287 # unnecessary ORDER BY clause.
1288 if (
1289 self.subquery
1290 and not connection.features.ignores_unnecessary_order_by_in_subqueries
1291 ):
1292 self.clear_ordering(force=False)
1293 for query in self.combined_queries:
1294 query.clear_ordering(force=False)
1295 sql, params = self.get_compiler(connection=connection).as_sql()
1296 if self.subquery:
1297 sql = "(%s)" % sql
1298 return sql, params
1299
1300 def resolve_lookup_value(self, value, can_reuse, allow_joins, summarize=False):
1301 if hasattr(value, "resolve_expression"):
1302 value = value.resolve_expression(
1303 self,
1304 reuse=can_reuse,
1305 allow_joins=allow_joins,
1306 summarize=summarize,
1307 )
1308 elif isinstance(value, (list, tuple)):
1309 # The items of the iterable may be expressions and therefore need
1310 # to be resolved independently.
1311 values = (
1312 self.resolve_lookup_value(sub_value, can_reuse, allow_joins, summarize)
1313 for sub_value in value
1314 )
1315 type_ = type(value)
1316 if hasattr(type_, "_make"): # namedtuple
1317 return type_(*values)
1318 return type_(values)
1319 return value
1320
1321 def solve_lookup_type(self, lookup, summarize=False):
1322 """
1323 Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains').
1324 """
1325 lookup_splitted = lookup.split(LOOKUP_SEP)
1326 if self.annotations:
1327 annotation, expression_lookups = refs_expression(
1328 lookup_splitted, self.annotations
1329 )
1330 if annotation:
1331 expression = self.annotations[annotation]
1332 if summarize:
1333 expression = Ref(annotation, expression)
1334 return expression_lookups, (), expression
1335 _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
1336 field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)]
1337 if len(lookup_parts) > 1 and not field_parts:
1338 raise FieldError(
1339 'Invalid lookup "%s" for model %s".'
1340 % (lookup, self.get_meta().model.__name__)
1341 )
1342 return lookup_parts, field_parts, False
1343
1344 def check_query_object_type(self, value, opts, field):
1345 """
1346 Check whether the object passed while querying is of the correct type.
1347 If not, raise a ValueError specifying the wrong object.
1348 """
1349 if hasattr(value, "_meta"):
1350 if not check_rel_lookup_compatibility(value._meta.model, opts, field):
1351 raise ValueError(
1352 'Cannot query "%s": Must be "%s" instance.'
1353 % (value, opts.object_name)
1354 )
1355
1356 def check_related_objects(self, field, value, opts):
1357 """Check the type of object passed to query relations."""
1358 if field.is_relation:
1359 # Check that the field and the queryset use the same model in a
1360 # query like .filter(author=Author.objects.all()). For example, the
1361 # opts would be Author's (from the author field) and value.model
1362 # would be Author.objects.all() queryset's .model (Author also).
1363 # The field is the related field on the lhs side.
1364 if (
1365 isinstance(value, Query)
1366 and not value.has_select_fields
1367 and not check_rel_lookup_compatibility(value.model, opts, field)
1368 ):
1369 raise ValueError(
1370 'Cannot use QuerySet for "%s": Use a QuerySet for "%s".'
1371 % (value.model._meta.object_name, opts.object_name)
1372 )
1373 elif hasattr(value, "_meta"):
1374 self.check_query_object_type(value, opts, field)
1375 elif hasattr(value, "__iter__"):
1376 for v in value:
1377 self.check_query_object_type(v, opts, field)
1378
1379 def check_filterable(self, expression):
1380 """Raise an error if expression cannot be used in a WHERE clause."""
1381 if hasattr(expression, "resolve_expression") and not getattr(
1382 expression, "filterable", True
1383 ):
1384 raise NotSupportedError(
1385 expression.__class__.__name__ + " is disallowed in the filter "
1386 "clause."
1387 )
1388 if hasattr(expression, "get_source_expressions"):
1389 for expr in expression.get_source_expressions():
1390 self.check_filterable(expr)
1391
1392 def build_lookup(self, lookups, lhs, rhs):
1393 """
1394 Try to extract transforms and lookup from given lhs.
1395
1396 The lhs value is something that works like SQLExpression.
1397 The rhs value is what the lookup is going to compare against.
1398 The lookups is a list of names to extract using get_lookup()
1399 and get_transform().
1400 """
1401 # __exact is the default lookup if one isn't given.
1402 *transforms, lookup_name = lookups or ["exact"]
1403 for name in transforms:
1404 lhs = self.try_transform(lhs, name, lookups)
1405 # First try get_lookup() so that the lookup takes precedence if the lhs
1406 # supports both transform and lookup for the name.
1407 lookup_class = lhs.get_lookup(lookup_name)
1408 if not lookup_class:
1409 # A lookup wasn't found. Try to interpret the name as a transform
1410 # and do an Exact lookup against it.
1411 lhs = self.try_transform(lhs, lookup_name)
1412 lookup_name = "exact"
1413 lookup_class = lhs.get_lookup(lookup_name)
1414 if not lookup_class:
1415 return
1416
1417 lookup = lookup_class(lhs, rhs)
1418 # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all
1419 # uses of None as a query value unless the lookup supports it.
1420 if lookup.rhs is None and not lookup.can_use_none_as_rhs:
1421 if lookup_name not in ("exact", "iexact"):
1422 raise ValueError("Cannot use None as a query value")
1423 return lhs.get_lookup("isnull")(lhs, True)
1424
1425 # For Oracle '' is equivalent to null. The check must be done at this
1426 # stage because join promotion can't be done in the compiler. Using
1427 # DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here.
1428 # A similar thing is done in is_nullable(), too.
1429 if (
1430 lookup_name == "exact"
1431 and lookup.rhs == ""
1432 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
1433 ):
1434 return lhs.get_lookup("isnull")(lhs, True)
1435
1436 return lookup
1437
1438 def try_transform(self, lhs, name, lookups=None):
1439 """
1440 Helper method for build_lookup(). Try to fetch and initialize
1441 a transform for name parameter from lhs.
1442 """
1443 transform_class = lhs.get_transform(name)
1444 if transform_class:
1445 return transform_class(lhs)
1446 else:
1447 output_field = lhs.output_field.__class__
1448 suggested_lookups = difflib.get_close_matches(
1449 name, lhs.output_field.get_lookups()
1450 )
1451 if suggested_lookups:
1452 suggestion = ", perhaps you meant %s?" % " or ".join(suggested_lookups)
1453 else:
1454 suggestion = "."
1455 if lookups is not None:
1456 name_index = lookups.index(name)
1457 unsupported_lookup = LOOKUP_SEP.join(lookups[name_index:])
1458 else:
1459 unsupported_lookup = name
1460 raise FieldError(
1461 "Unsupported lookup '%s' for %s or join on the field not "
1462 "permitted%s" % (unsupported_lookup, output_field.__name__, suggestion)
1463 )
1464
1465 def build_filter(
1466 self,
1467 filter_expr,
1468 branch_negated=False,
1469 current_negated=False,
1470 can_reuse=None,
1471 allow_joins=True,
1472 split_subq=True,
1473 check_filterable=True,
1474 summarize=False,
1475 update_join_types=True,
1476 ):
1477 """
1478 Build a WhereNode for a single filter clause but don't add it
1479 to this Query. Query.add_q() will then add this filter to the where
1480 Node.
1481
1482 The 'branch_negated' tells us if the current branch contains any
1483 negations. This will be used to determine if subqueries are needed.
1484
1485 The 'current_negated' is used to determine if the current filter is
1486 negated or not and this will be used to determine if IS NULL filtering
1487 is needed.
1488
1489 The difference between current_negated and branch_negated is that
1490 branch_negated is set on first negation, but current_negated is
1491 flipped for each negation.
1492
1493 Note that add_filter will not do any negating itself, that is done
1494 upper in the code by add_q().
1495
1496 The 'can_reuse' is a set of reusable joins for multijoins.
1497
1498 The method will create a filter clause that can be added to the current
1499 query. However, if the filter isn't added to the query then the caller
1500 is responsible for unreffing the joins used.
1501 """
1502 if isinstance(filter_expr, dict):
1503 raise FieldError("Cannot parse keyword query as dict")
1504 if isinstance(filter_expr, Q):
1505 return self._add_q(
1506 filter_expr,
1507 branch_negated=branch_negated,
1508 current_negated=current_negated,
1509 used_aliases=can_reuse,
1510 allow_joins=allow_joins,
1511 split_subq=split_subq,
1512 check_filterable=check_filterable,
1513 summarize=summarize,
1514 update_join_types=update_join_types,
1515 )
1516 if hasattr(filter_expr, "resolve_expression"):
1517 if not getattr(filter_expr, "conditional", False):
1518 raise TypeError("Cannot filter against a non-conditional expression.")
1519 condition = filter_expr.resolve_expression(
1520 self, allow_joins=allow_joins, reuse=can_reuse, summarize=summarize
1521 )
1522 if not isinstance(condition, Lookup):
1523 condition = self.build_lookup(["exact"], condition, True)
1524 return WhereNode([condition], connector=AND), []
1525 arg, value = filter_expr
1526 if not arg:
1527 raise FieldError("Cannot parse keyword query %r" % arg)
1528 lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize)
1529
1530 if check_filterable:
1531 self.check_filterable(reffed_expression)
1532
1533 if not allow_joins and len(parts) > 1:
1534 raise FieldError("Joined field references are not permitted in this query")
1535
1536 pre_joins = self.alias_refcount.copy()
1537 value = self.resolve_lookup_value(value, can_reuse, allow_joins, summarize)
1538 used_joins = {
1539 k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)
1540 }
1541
1542 if check_filterable:
1543 self.check_filterable(value)
1544
1545 if reffed_expression:
1546 condition = self.build_lookup(lookups, reffed_expression, value)
1547 return WhereNode([condition], connector=AND), []
1548
1549 opts = self.get_meta()
1550 alias = self.get_initial_alias()
1551 allow_many = not branch_negated or not split_subq
1552
1553 try:
1554 join_info = self.setup_joins(
1555 parts,
1556 opts,
1557 alias,
1558 can_reuse=can_reuse,
1559 allow_many=allow_many,
1560 )
1561
1562 # Prevent iterator from being consumed by check_related_objects()
1563 if isinstance(value, Iterator):
1564 value = list(value)
1565 self.check_related_objects(join_info.final_field, value, join_info.opts)
1566
1567 # split_exclude() needs to know which joins were generated for the
1568 # lookup parts
1569 self._lookup_joins = join_info.joins
1570 except MultiJoin as e:
1571 return self.split_exclude(filter_expr, can_reuse, e.names_with_path)
1572
1573 # Update used_joins before trimming since they are reused to determine
1574 # which joins could be later promoted to INNER.
1575 used_joins.update(join_info.joins)
1576 targets, alias, join_list = self.trim_joins(
1577 join_info.targets, join_info.joins, join_info.path
1578 )
1579 if can_reuse is not None:
1580 can_reuse.update(join_list)
1581
1582 if join_info.final_field.is_relation:
1583 if len(targets) == 1:
1584 col = self._get_col(targets[0], join_info.final_field, alias)
1585 else:
1586 col = ColPairs(alias, targets, join_info.targets, join_info.final_field)
1587 else:
1588 col = self._get_col(targets[0], join_info.final_field, alias)
1589
1590 condition = self.build_lookup(lookups, col, value)
1591 lookup_type = condition.lookup_name
1592 clause = WhereNode([condition], connector=AND)
1593
1594 require_outer = (
1595 lookup_type == "isnull" and condition.rhs is True and not current_negated
1596 )
1597 if (
1598 current_negated
1599 and (lookup_type != "isnull" or condition.rhs is False)
1600 and condition.rhs is not None
1601 ):
1602 require_outer = True
1603 if lookup_type != "isnull":
1604 # The condition added here will be SQL like this:
1605 # NOT (col IS NOT NULL), where the first NOT is added in
1606 # upper layers of code. The reason for addition is that if col
1607 # is null, then col != someval will result in SQL "unknown"
1608 # which isn't the same as in Python. The Python None handling
1609 # is wanted, and it can be gotten by
1610 # (col IS NULL OR col != someval)
1611 # <=>
1612 # NOT (col IS NOT NULL AND col = someval).
1613 if (
1614 self.is_nullable(targets[0])
1615 or self.alias_map[join_list[-1]].join_type == LOUTER
1616 ):
1617 lookup_class = targets[0].get_lookup("isnull")
1618 col = self._get_col(targets[0], join_info.targets[0], alias)
1619 clause.add(lookup_class(col, False), AND)
1620 # If someval is a nullable column, someval IS NOT NULL is
1621 # added.
1622 if isinstance(value, Col) and self.is_nullable(value.target):
1623 lookup_class = value.target.get_lookup("isnull")
1624 clause.add(lookup_class(value, False), AND)
1625 return clause, used_joins if not require_outer else ()
1626
1627 def add_filter(self, filter_lhs, filter_rhs):
1628 self.add_q(Q((filter_lhs, filter_rhs)))
1629
1630 def add_q(self, q_object, reuse_all=False):
1631 """
1632 A preprocessor for the internal _add_q(). Responsible for doing final
1633 join promotion.
1634 """
1635 # For join promotion this case is doing an AND for the added q_object
1636 # and existing conditions. So, any existing inner join forces the join
1637 # type to remain inner. Existing outer joins can however be demoted.
1638 # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if
1639 # rel_a doesn't produce any rows, then the whole condition must fail.
1640 # So, demotion is OK.
1641 existing_inner = {
1642 a for a in self.alias_map if self.alias_map[a].join_type == INNER
1643 }
1644 if reuse_all:
1645 can_reuse = set(self.alias_map)
1646 else:
1647 can_reuse = self.used_aliases
1648 clause, _ = self._add_q(q_object, can_reuse)
1649 if clause:
1650 self.where.add(clause, AND)
1651 self.demote_joins(existing_inner)
1652
1653 def build_where(self, filter_expr):
1654 return self.build_filter(filter_expr, allow_joins=False)[0]
1655
1656 def clear_where(self):
1657 self.where = WhereNode()
1658
1659 def _add_q(
1660 self,
1661 q_object,
1662 used_aliases,
1663 branch_negated=False,
1664 current_negated=False,
1665 allow_joins=True,
1666 split_subq=True,
1667 check_filterable=True,
1668 summarize=False,
1669 update_join_types=True,
1670 ):
1671 """Add a Q-object to the current filter."""
1672 connector = q_object.connector
1673 current_negated ^= q_object.negated
1674 branch_negated = branch_negated or q_object.negated
1675 target_clause = WhereNode(connector=connector, negated=q_object.negated)
1676 joinpromoter = JoinPromoter(
1677 q_object.connector, len(q_object.children), current_negated
1678 )
1679 for child in q_object.children:
1680 child_clause, needed_inner = self.build_filter(
1681 child,
1682 can_reuse=used_aliases,
1683 branch_negated=branch_negated,
1684 current_negated=current_negated,
1685 allow_joins=allow_joins,
1686 split_subq=split_subq,
1687 check_filterable=check_filterable,
1688 summarize=summarize,
1689 update_join_types=update_join_types,
1690 )
1691 joinpromoter.add_votes(needed_inner)
1692 if child_clause:
1693 target_clause.add(child_clause, connector)
1694 if update_join_types:
1695 needed_inner = joinpromoter.update_join_types(self)
1696 else:
1697 needed_inner = []
1698 return target_clause, needed_inner
1699
1700 def add_filtered_relation(self, filtered_relation, alias):
1701 if "." in alias:
1702 raise ValueError(
1703 "FilteredRelation doesn't support aliases with periods "
1704 "(got %r)." % alias
1705 )
1706 self.check_alias(alias)
1707 filtered_relation.alias = alias
1708 relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type(
1709 filtered_relation.relation_name
1710 )
1711 if relation_lookup_parts:
1712 raise ValueError(
1713 "FilteredRelation's relation_name cannot contain lookups "
1714 "(got %r)." % filtered_relation.relation_name
1715 )
1716 for lookup in get_children_from_q(filtered_relation.condition):
1717 lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup)
1718 shift = 2 if not lookup_parts else 1
1719 lookup_field_path = lookup_field_parts[:-shift]
1720 for idx, lookup_field_part in enumerate(lookup_field_path):
1721 if len(relation_field_parts) > idx:
1722 if relation_field_parts[idx] != lookup_field_part:
1723 raise ValueError(
1724 "FilteredRelation's condition doesn't support "
1725 "relations outside the %r (got %r)."
1726 % (filtered_relation.relation_name, lookup)
1727 )
1728 if len(lookup_field_parts) > len(relation_field_parts) + 1:
1729 raise ValueError(
1730 "FilteredRelation's condition doesn't support nested "
1731 "relations deeper than the relation_name (got %r for "
1732 "%r)." % (lookup, filtered_relation.relation_name)
1733 )
1734 filtered_relation.condition = rename_prefix_from_q(
1735 filtered_relation.relation_name,
1736 alias,
1737 filtered_relation.condition,
1738 )
1739 self._filtered_relations[filtered_relation.alias] = filtered_relation
1740
1741 def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
1742 """
1743 Walk the list of names and turns them into PathInfo tuples. A single
1744 name in 'names' can generate multiple PathInfos (m2m, for example).
1745
1746 'names' is the path of names to travel, 'opts' is the model Options we
1747 start the name resolving from, 'allow_many' is as for setup_joins().
1748 If fail_on_missing is set to True, then a name that can't be resolved
1749 will generate a FieldError.
1750
1751 Return a list of PathInfo tuples. In addition return the final field
1752 (the last used join field) and target (which is a field guaranteed to
1753 contain the same value as the final field). Finally, return those names
1754 that weren't found (which are likely transforms and the final lookup).
1755 """
1756 path, names_with_path = [], []
1757 for pos, name in enumerate(names):
1758 cur_names_with_path = (name, [])
1759 if name == "pk" and opts is not None:
1760 name = opts.pk.name
1761
1762 field = None
1763 filtered_relation = None
1764 try:
1765 if opts is None:
1766 raise FieldDoesNotExist
1767 field = opts.get_field(name)
1768 except FieldDoesNotExist:
1769 if name in self.annotation_select:
1770 field = self.annotation_select[name].output_field
1771 elif name in self._filtered_relations and pos == 0:
1772 filtered_relation = self._filtered_relations[name]
1773 if LOOKUP_SEP in filtered_relation.relation_name:
1774 parts = filtered_relation.relation_name.split(LOOKUP_SEP)
1775 filtered_relation_path, field, _, _ = self.names_to_path(
1776 parts,
1777 opts,
1778 allow_many,
1779 fail_on_missing,
1780 )
1781 path.extend(filtered_relation_path[:-1])
1782 else:
1783 field = opts.get_field(filtered_relation.relation_name)
1784 if field is not None:
1785 # Fields that contain one-to-many relations with a generic
1786 # model (like a GenericForeignKey) cannot generate reverse
1787 # relations and therefore cannot be used for reverse querying.
1788 if field.is_relation and not field.related_model:
1789 raise FieldError(
1790 "Field %r does not generate an automatic reverse "
1791 "relation and therefore cannot be used for reverse "
1792 "querying. If it is a GenericForeignKey, consider "
1793 "adding a GenericRelation." % name
1794 )
1795 try:
1796 model = field.model._meta.concrete_model
1797 except AttributeError:
1798 # QuerySet.annotate() may introduce fields that aren't
1799 # attached to a model.
1800 model = None
1801 else:
1802 # We didn't find the current field, so move position back
1803 # one step.
1804 pos -= 1
1805 if pos == -1 or fail_on_missing:
1806 available = sorted(
1807 [
1808 *get_field_names_from_opts(opts),
1809 *self.annotation_select,
1810 *self._filtered_relations,
1811 ]
1812 )
1813 raise FieldError(
1814 "Cannot resolve keyword '%s' into field. "
1815 "Choices are: %s" % (name, ", ".join(available))
1816 )
1817 break
1818 # Check if we need any joins for concrete inheritance cases (the
1819 # field lives in parent, but we are currently in one of its
1820 # children)
1821 if opts is not None and model is not opts.model:
1822 path_to_parent = opts.get_path_to_parent(model)
1823 if path_to_parent:
1824 path.extend(path_to_parent)
1825 cur_names_with_path[1].extend(path_to_parent)
1826 opts = path_to_parent[-1].to_opts
1827 if hasattr(field, "path_infos"):
1828 if filtered_relation:
1829 pathinfos = field.get_path_info(filtered_relation)
1830 else:
1831 pathinfos = field.path_infos
1832 if not allow_many:
1833 for inner_pos, p in enumerate(pathinfos):
1834 if p.m2m:
1835 cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1])
1836 names_with_path.append(cur_names_with_path)
1837 raise MultiJoin(pos + 1, names_with_path)
1838 last = pathinfos[-1]
1839 path.extend(pathinfos)
1840 final_field = last.join_field
1841 opts = last.to_opts
1842 targets = last.target_fields
1843 cur_names_with_path[1].extend(pathinfos)
1844 names_with_path.append(cur_names_with_path)
1845 else:
1846 # Local non-relational field.
1847 final_field = field
1848 targets = (field,)
1849 if fail_on_missing and pos + 1 != len(names):
1850 raise FieldError(
1851 "Cannot resolve keyword %r into field. Join on '%s'"
1852 " not permitted." % (names[pos + 1], name)
1853 )
1854 break
1855 return path, final_field, targets, names[pos + 1 :]
1856
1857 def setup_joins(
1858 self,
1859 names,
1860 opts,
1861 alias,
1862 can_reuse=None,
1863 allow_many=True,
1864 ):
1865 """
1866 Compute the necessary table joins for the passage through the fields
1867 given in 'names'. 'opts' is the Options class for the current model
1868 (which gives the table we are starting from), 'alias' is the alias for
1869 the table to start the joining from.
1870
1871 The 'can_reuse' defines the reverse foreign key joins we can reuse. It
1872 can be None in which case all joins are reusable or a set of aliases
1873 that can be reused. Note that non-reverse foreign keys are always
1874 reusable when using setup_joins().
1875
1876 If 'allow_many' is False, then any reverse foreign key seen will
1877 generate a MultiJoin exception.
1878
1879 Return the final field involved in the joins, the target field (used
1880 for any 'where' constraint), the final 'opts' value, the joins, the
1881 field path traveled to generate the joins, and a transform function
1882 that takes a field and alias and is equivalent to `field.get_col(alias)`
1883 in the simple case but wraps field transforms if they were included in
1884 names.
1885
1886 The target field is the field containing the concrete value. Final
1887 field can be something different, for example foreign key pointing to
1888 that value. Final field is needed for example in some value
1889 conversions (convert 'obj' in fk__id=obj to pk val using the foreign
1890 key field for example).
1891 """
1892 joins = [alias]
1893 # The transform can't be applied yet, as joins must be trimmed later.
1894 # To avoid making every caller of this method look up transforms
1895 # directly, compute transforms here and create a partial that converts
1896 # fields to the appropriate wrapped version.
1897
1898 def final_transformer(field, alias):
1899 if not self.alias_cols:
1900 alias = None
1901 return field.get_col(alias)
1902
1903 # Try resolving all the names as fields first. If there's an error,
1904 # treat trailing names as lookups until a field can be resolved.
1905 last_field_exception = None
1906 for pivot in range(len(names), 0, -1):
1907 try:
1908 path, final_field, targets, rest = self.names_to_path(
1909 names[:pivot],
1910 opts,
1911 allow_many,
1912 fail_on_missing=True,
1913 )
1914 except FieldError as exc:
1915 if pivot == 1:
1916 # The first item cannot be a lookup, so it's safe
1917 # to raise the field error here.
1918 raise
1919 else:
1920 last_field_exception = exc
1921 else:
1922 # The transforms are the remaining items that couldn't be
1923 # resolved into fields.
1924 transforms = names[pivot:]
1925 break
1926 for name in transforms:
1927
1928 def transform(field, alias, *, name, previous):
1929 try:
1930 wrapped = previous(field, alias)
1931 return self.try_transform(wrapped, name)
1932 except FieldError:
1933 # FieldError is raised if the transform doesn't exist.
1934 if isinstance(final_field, Field) and last_field_exception:
1935 raise last_field_exception
1936 else:
1937 raise
1938
1939 final_transformer = functools.partial(
1940 transform, name=name, previous=final_transformer
1941 )
1942 final_transformer.has_transforms = True
1943 # Then, add the path to the query's joins. Note that we can't trim
1944 # joins at this stage - we will need the information about join type
1945 # of the trimmed joins.
1946 for join in path:
1947 if join.filtered_relation:
1948 filtered_relation = join.filtered_relation.clone()
1949 table_alias = filtered_relation.alias
1950 else:
1951 filtered_relation = None
1952 table_alias = None
1953 opts = join.to_opts
1954 if join.direct:
1955 nullable = self.is_nullable(join.join_field)
1956 else:
1957 nullable = True
1958 connection = self.join_class(
1959 opts.db_table,
1960 alias,
1961 table_alias,
1962 INNER,
1963 join.join_field,
1964 nullable,
1965 filtered_relation=filtered_relation,
1966 )
1967 reuse = can_reuse if join.m2m else None
1968 alias = self.join(connection, reuse=reuse)
1969 joins.append(alias)
1970 if join.filtered_relation and can_reuse is not None:
1971 can_reuse.add(alias)
1972 return JoinInfo(final_field, targets, opts, joins, path, final_transformer)
1973
1974 def trim_joins(self, targets, joins, path):
1975 """
1976 The 'target' parameter is the final field being joined to, 'joins'
1977 is the full list of join aliases. The 'path' contain the PathInfos
1978 used to create the joins.
1979
1980 Return the final target field and table alias and the new active
1981 joins.
1982
1983 Always trim any direct join if the target column is already in the
1984 previous table. Can't trim reverse joins as it's unknown if there's
1985 anything on the other side of the join.
1986 """
1987 joins = joins[:]
1988 for pos, info in enumerate(reversed(path)):
1989 if len(joins) == 1 or not info.direct:
1990 break
1991 if info.filtered_relation:
1992 break
1993 join_targets = {t.column for t in info.join_field.foreign_related_fields}
1994 cur_targets = {t.column for t in targets}
1995 if not cur_targets.issubset(join_targets):
1996 break
1997 targets_dict = {
1998 r[1].column: r[0]
1999 for r in info.join_field.related_fields
2000 if r[1].column in cur_targets
2001 }
2002 targets = tuple(targets_dict[t.column] for t in targets)
2003 self.unref_alias(joins.pop())
2004 return targets, joins[-1], joins
2005
2006 @classmethod
2007 def _gen_cols(cls, exprs, include_external=False, resolve_refs=True):
2008 for expr in exprs:
2009 if isinstance(expr, Col):
2010 yield expr
2011 elif include_external and callable(
2012 getattr(expr, "get_external_cols", None)
2013 ):
2014 yield from expr.get_external_cols()
2015 elif hasattr(expr, "get_source_expressions"):
2016 if not resolve_refs and isinstance(expr, Ref):
2017 continue
2018 yield from cls._gen_cols(
2019 expr.get_source_expressions(),
2020 include_external=include_external,
2021 resolve_refs=resolve_refs,
2022 )
2023
2024 @classmethod
2025 def _gen_col_aliases(cls, exprs):
2026 yield from (expr.alias for expr in cls._gen_cols(exprs))
2027
2028 def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
2029 annotation = self.annotations.get(name)
2030 if annotation is not None:
2031 if not allow_joins:
2032 for alias in self._gen_col_aliases([annotation]):
2033 if isinstance(self.alias_map[alias], Join):
2034 raise FieldError(
2035 "Joined field references are not permitted in this query"
2036 )
2037 if summarize:
2038 # Summarize currently means we are doing an aggregate() query
2039 # which is executed as a wrapped subquery if any of the
2040 # aggregate() elements reference an existing annotation. In
2041 # that case we need to return a Ref to the subquery's annotation.
2042 if name not in self.annotation_select:
2043 raise FieldError(
2044 "Cannot aggregate over the '%s' alias. Use annotate() "
2045 "to promote it." % name
2046 )
2047 return Ref(name, self.annotation_select[name])
2048 else:
2049 return annotation
2050 else:
2051 field_list = name.split(LOOKUP_SEP)
2052 annotation = self.annotations.get(field_list[0])
2053 if annotation is not None:
2054 for transform in field_list[1:]:
2055 annotation = self.try_transform(annotation, transform)
2056 return annotation
2057 join_info = self.setup_joins(
2058 field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse
2059 )
2060 targets, final_alias, join_list = self.trim_joins(
2061 join_info.targets, join_info.joins, join_info.path
2062 )
2063 if not allow_joins and len(join_list) > 1:
2064 raise FieldError(
2065 "Joined field references are not permitted in this query"
2066 )
2067 if len(targets) > 1:
2068 raise FieldError(
2069 "Referencing multicolumn fields with F() objects isn't supported"
2070 )
2071 # Verify that the last lookup in name is a field or a transform:
2072 # transform_function() raises FieldError if not.
2073 transform = join_info.transform_function(targets[0], final_alias)
2074 if reuse is not None:
2075 reuse.update(join_list)
2076 return transform
2077
2078 def split_exclude(self, filter_expr, can_reuse, names_with_path):
2079 """
2080 When doing an exclude against any kind of N-to-many relation, we need
2081 to use a subquery. This method constructs the nested query, given the
2082 original exclude filter (filter_expr) and the portion up to the first
2083 N-to-many relation field.
2084
2085 For example, if the origin filter is ~Q(child__name='foo'), filter_expr
2086 is ('child__name', 'foo') and can_reuse is a set of joins usable for
2087 filters in the original query.
2088
2089 We will turn this into equivalent of:
2090 WHERE NOT EXISTS(
2091 SELECT 1
2092 FROM child
2093 WHERE name = 'foo' AND child.parent_id = parent.id
2094 LIMIT 1
2095 )
2096 """
2097 # Generate the inner query.
2098 query = self.__class__(self.model)
2099 query._filtered_relations = self._filtered_relations
2100 filter_lhs, filter_rhs = filter_expr
2101 if isinstance(filter_rhs, OuterRef):
2102 filter_rhs = OuterRef(filter_rhs)
2103 elif isinstance(filter_rhs, F):
2104 filter_rhs = OuterRef(filter_rhs.name)
2105 query.add_filter(filter_lhs, filter_rhs)
2106 query.clear_ordering(force=True)
2107 # Try to have as simple as possible subquery -> trim leading joins from
2108 # the subquery.
2109 trimmed_prefix, contains_louter = query.trim_start(names_with_path)
2110
2111 col = query.select[0]
2112 select_field = col.target
2113 alias = col.alias
2114 if alias in can_reuse:
2115 pk = select_field.model._meta.pk
2116 # Need to add a restriction so that outer query's filters are in effect for
2117 # the subquery, too.
2118 query.bump_prefix(self)
2119 lookup_class = select_field.get_lookup("exact")
2120 # Note that the query.select[0].alias is different from alias
2121 # due to bump_prefix above.
2122 lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias))
2123 query.where.add(lookup, AND)
2124 query.external_aliases[alias] = True
2125 else:
2126 lookup_class = select_field.get_lookup("exact")
2127 lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix))
2128 query.where.add(lookup, AND)
2129
2130 condition, needed_inner = self.build_filter(Exists(query))
2131
2132 if contains_louter:
2133 or_null_condition, _ = self.build_filter(
2134 ("%s__isnull" % trimmed_prefix, True),
2135 current_negated=True,
2136 branch_negated=True,
2137 can_reuse=can_reuse,
2138 )
2139 condition.add(or_null_condition, OR)
2140 # Note that the end result will be:
2141 # NOT EXISTS (inner_q) OR outercol IS NULL
2142 # this might look crazy but due to how NULL works, this seems to be
2143 # correct. If the IS NULL check is removed, then if outercol
2144 # IS NULL we will not match the row.
2145 return condition, needed_inner
2146
2147 def set_empty(self):
2148 self.where.add(NothingNode(), AND)
2149 for query in self.combined_queries:
2150 query.set_empty()
2151
2152 def is_empty(self):
2153 return any(isinstance(c, NothingNode) for c in self.where.children)
2154
2155 def set_limits(self, low=None, high=None):
2156 """
2157 Adjust the limits on the rows retrieved. Use low/high to set these,
2158 as it makes it more Pythonic to read and write. When the SQL query is
2159 created, convert them to the appropriate offset and limit values.
2160
2161 Apply any limits passed in here to the existing constraints. Add low
2162 to the current low value and clamp both to any existing high value.
2163 """
2164 if high is not None:
2165 if self.high_mark is not None:
2166 self.high_mark = min(self.high_mark, self.low_mark + high)
2167 else:
2168 self.high_mark = self.low_mark + high
2169 if low is not None:
2170 if self.high_mark is not None:
2171 self.low_mark = min(self.high_mark, self.low_mark + low)
2172 else:
2173 self.low_mark = self.low_mark + low
2174
2175 if self.low_mark == self.high_mark:
2176 self.set_empty()
2177
2178 def clear_limits(self):
2179 """Clear any existing limits."""
2180 self.low_mark, self.high_mark = 0, None
2181
2182 @property
2183 def is_sliced(self):
2184 return self.low_mark != 0 or self.high_mark is not None
2185
2186 def has_limit_one(self):
2187 return self.high_mark is not None and (self.high_mark - self.low_mark) == 1
2188
2189 def can_filter(self):
2190 """
2191 Return True if adding filters to this instance is still possible.
2192
2193 Typically, this means no limits or offsets have been put on the results.
2194 """
2195 return not self.is_sliced
2196
2197 def clear_select_clause(self):
2198 """Remove all fields from SELECT clause."""
2199 self.select = ()
2200 self.default_cols = False
2201 self.select_related = False
2202 self.set_extra_mask(())
2203 self.set_annotation_mask(())
2204 self.selected = None
2205
2206 def clear_select_fields(self):
2207 """
2208 Clear the list of fields to select (but not extra_select columns).
2209 Some queryset types completely replace any existing list of select
2210 columns.
2211 """
2212 self.select = ()
2213 self.values_select = ()
2214 self.selected = None
2215
2216 def add_select_col(self, col, name):
2217 self.select += (col,)
2218 self.values_select += (name,)
2219 self.selected[name] = len(self.select) - 1
2220
2221 def set_select(self, cols):
2222 self.default_cols = False
2223 self.select = tuple(cols)
2224
2225 def add_distinct_fields(self, *field_names):
2226 """
2227 Add and resolve the given fields to the query's "distinct on" clause.
2228 """
2229 self.distinct_fields = field_names
2230 self.distinct = True
2231
2232 def add_fields(self, field_names, allow_m2m=True):
2233 """
2234 Add the given (model) fields to the select set. Add the field names in
2235 the order specified.
2236 """
2237 alias = self.get_initial_alias()
2238 opts = self.get_meta()
2239
2240 try:
2241 cols = []
2242 for name in field_names:
2243 # Join promotion note - we must not remove any rows here, so
2244 # if there is no existing joins, use outer join.
2245 join_info = self.setup_joins(
2246 name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m
2247 )
2248 targets, final_alias, joins = self.trim_joins(
2249 join_info.targets,
2250 join_info.joins,
2251 join_info.path,
2252 )
2253 if len(targets) > 1:
2254 transformed_targets = [
2255 join_info.transform_function(target, final_alias)
2256 for target in targets
2257 ]
2258 cols.append(
2259 ColPairs(
2260 final_alias if self.alias_cols else None,
2261 [col.target for col in transformed_targets],
2262 [col.output_field for col in transformed_targets],
2263 join_info.final_field,
2264 )
2265 )
2266 else:
2267 cols.append(join_info.transform_function(targets[0], final_alias))
2268 if cols:
2269 self.set_select(cols)
2270 except MultiJoin:
2271 raise FieldError("Invalid field name: '%s'" % name)
2272 except FieldError:
2273 if LOOKUP_SEP in name:
2274 # For lookups spanning over relationships, show the error
2275 # from the model on which the lookup failed.
2276 raise
2277 else:
2278 names = sorted(
2279 [
2280 *get_field_names_from_opts(opts),
2281 *self.extra,
2282 *self.annotation_select,
2283 *self._filtered_relations,
2284 ]
2285 )
2286 raise FieldError(
2287 "Cannot resolve keyword %r into field. "
2288 "Choices are: %s" % (name, ", ".join(names))
2289 )
2290
2291 def add_ordering(self, *ordering):
2292 """
2293 Add items from the 'ordering' sequence to the query's "order by"
2294 clause. These items are either field names (not column names) --
2295 possibly with a direction prefix ('-' or '?') -- or OrderBy
2296 expressions.
2297
2298 If 'ordering' is empty, clear all ordering from the query.
2299 """
2300 errors = []
2301 for item in ordering:
2302 if isinstance(item, str):
2303 if item == "?":
2304 continue
2305 item = item.removeprefix("-")
2306 if item in self.annotations:
2307 continue
2308 if self.extra and item in self.extra:
2309 continue
2310 # names_to_path() validates the lookup. A descriptive
2311 # FieldError will be raise if it's not.
2312 self.names_to_path(item.split(LOOKUP_SEP), self.model._meta)
2313 elif not hasattr(item, "resolve_expression"):
2314 errors.append(item)
2315 if getattr(item, "contains_aggregate", False):
2316 raise FieldError(
2317 "Using an aggregate in order_by() without also including "
2318 "it in annotate() is not allowed: %s" % item
2319 )
2320 if errors:
2321 raise FieldError("Invalid order_by arguments: %s" % errors)
2322 if ordering:
2323 self.order_by += ordering
2324 else:
2325 self.default_ordering = False
2326
2327 def clear_ordering(self, force=False, clear_default=True):
2328 """
2329 Remove any ordering settings if the current query allows it without
2330 side effects, set 'force' to True to clear the ordering regardless.
2331 If 'clear_default' is True, there will be no ordering in the resulting
2332 query (not even the model's default).
2333 """
2334 if not force and (
2335 self.is_sliced or self.distinct_fields or self.select_for_update
2336 ):
2337 return
2338 self.order_by = ()
2339 self.extra_order_by = ()
2340 if clear_default:
2341 self.default_ordering = False
2342
2343 def set_group_by(self, allow_aliases=True):
2344 """
2345 Expand the GROUP BY clause required by the query.
2346
2347 This will usually be the set of all non-aggregate fields in the
2348 return data. If the database backend supports grouping by the
2349 primary key, and the query would be equivalent, the optimization
2350 will be made automatically.
2351 """
2352 if allow_aliases and self.values_select:
2353 # If grouping by aliases is allowed assign selected value aliases
2354 # by moving them to annotations.
2355 group_by_annotations = {}
2356 values_select = {}
2357 for alias, expr in zip(self.values_select, self.select):
2358 if isinstance(expr, Col):
2359 values_select[alias] = expr
2360 else:
2361 group_by_annotations[alias] = expr
2362 self.annotations = {**group_by_annotations, **self.annotations}
2363 self.append_annotation_mask(group_by_annotations)
2364 self.select = tuple(values_select.values())
2365 self.values_select = tuple(values_select)
2366 if self.selected is not None:
2367 for index, value_select in enumerate(values_select):
2368 self.selected[value_select] = index
2369 group_by = list(self.select)
2370 for alias, annotation in self.annotation_select.items():
2371 if not (group_by_cols := annotation.get_group_by_cols()):
2372 continue
2373 if allow_aliases and not annotation.contains_aggregate:
2374 group_by.append(Ref(alias, annotation))
2375 else:
2376 group_by.extend(group_by_cols)
2377 self.group_by = tuple(group_by)
2378
2379 def add_select_related(self, fields):
2380 """
2381 Set up the select_related data structure so that we only select
2382 certain related models (as opposed to all models, when
2383 self.select_related=True).
2384 """
2385 if isinstance(self.select_related, bool):
2386 field_dict = {}
2387 else:
2388 field_dict = self.select_related
2389 for field in fields:
2390 d = field_dict
2391 for part in field.split(LOOKUP_SEP):
2392 d = d.setdefault(part, {})
2393 self.select_related = field_dict
2394
2395 def add_extra(self, select, select_params, where, params, tables, order_by):
2396 """
2397 Add data to the various extra_* attributes for user-created additions
2398 to the query.
2399 """
2400 if select:
2401 # We need to pair any placeholder markers in the 'select'
2402 # dictionary with their parameters in 'select_params' so that
2403 # subsequent updates to the select dictionary also adjust the
2404 # parameters appropriately.
2405 select_pairs = {}
2406 if select_params:
2407 param_iter = iter(select_params)
2408 else:
2409 param_iter = iter([])
2410 for name, entry in select.items():
2411 self.check_alias(name)
2412 entry = str(entry)
2413 entry_params = []
2414 pos = entry.find("%s")
2415 while pos != -1:
2416 if pos == 0 or entry[pos - 1] != "%":
2417 entry_params.append(next(param_iter))
2418 pos = entry.find("%s", pos + 2)
2419 select_pairs[name] = (entry, entry_params)
2420 self.extra.update(select_pairs)
2421 if where or params:
2422 self.where.add(ExtraWhere(where, params), AND)
2423 if tables:
2424 self.extra_tables += tuple(tables)
2425 if order_by:
2426 self.extra_order_by = order_by
2427
2428 def clear_deferred_loading(self):
2429 """Remove any fields from the deferred loading set."""
2430 self.deferred_loading = (frozenset(), True)
2431
2432 def add_deferred_loading(self, field_names):
2433 """
2434 Add the given list of model field names to the set of fields to
2435 exclude from loading from the database when automatic column selection
2436 is done. Add the new field names to any existing field names that
2437 are deferred (or removed from any existing field names that are marked
2438 as the only ones for immediate loading).
2439 """
2440 # Fields on related models are stored in the literal double-underscore
2441 # format, so that we can use a set datastructure. We do the foo__bar
2442 # splitting and handling when computing the SQL column names (as part of
2443 # get_columns()).
2444 existing, defer = self.deferred_loading
2445 if defer:
2446 # Add to existing deferred names.
2447 self.deferred_loading = existing.union(field_names), True
2448 else:
2449 # Remove names from the set of any existing "immediate load" names.
2450 if new_existing := existing.difference(field_names):
2451 self.deferred_loading = new_existing, False
2452 else:
2453 self.clear_deferred_loading()
2454 if new_only := set(field_names).difference(existing):
2455 self.deferred_loading = new_only, True
2456
2457 def add_immediate_loading(self, field_names):
2458 """
2459 Add the given list of model field names to the set of fields to
2460 retrieve when the SQL is executed ("immediate loading" fields). The
2461 field names replace any existing immediate loading field names. If
2462 there are field names already specified for deferred loading, remove
2463 those names from the new field_names before storing the new names
2464 for immediate loading. (That is, immediate loading overrides any
2465 existing immediate values, but respects existing deferrals.)
2466 """
2467 existing, defer = self.deferred_loading
2468 field_names = set(field_names)
2469 if "pk" in field_names:
2470 field_names.remove("pk")
2471 field_names.add(self.get_meta().pk.name)
2472
2473 if defer:
2474 # Remove any existing deferred names from the current set before
2475 # setting the new names.
2476 self.deferred_loading = field_names.difference(existing), False
2477 else:
2478 # Replace any existing "immediate load" field names.
2479 self.deferred_loading = frozenset(field_names), False
2480
2481 def set_annotation_mask(self, names):
2482 """Set the mask of annotations that will be returned by the SELECT."""
2483 if names is None:
2484 self.annotation_select_mask = None
2485 else:
2486 self.annotation_select_mask = set(names)
2487 if self.selected:
2488 # Prune the masked annotations.
2489 self.selected = {
2490 key: value
2491 for key, value in self.selected.items()
2492 if not isinstance(value, str)
2493 or value in self.annotation_select_mask
2494 }
2495 # Append the unmasked annotations.
2496 for name in names:
2497 self.selected[name] = name
2498 self._annotation_select_cache = None
2499
2500 def append_annotation_mask(self, names):
2501 if self.annotation_select_mask is not None:
2502 self.set_annotation_mask(self.annotation_select_mask.union(names))
2503
2504 def set_extra_mask(self, names):
2505 """
2506 Set the mask of extra select items that will be returned by SELECT.
2507 Don't remove them from the Query since they might be used later.
2508 """
2509 if names is None:
2510 self.extra_select_mask = None
2511 else:
2512 self.extra_select_mask = set(names)
2513 self._extra_select_cache = None
2514
2515 @property
2516 def has_select_fields(self):
2517 return self.selected is not None
2518
2519 def set_values(self, fields):
2520 self.select_related = False
2521 self.clear_deferred_loading()
2522 self.clear_select_fields()
2523
2524 selected = {}
2525 if fields:
2526 for field in fields:
2527 self.check_alias(field)
2528 field_names = []
2529 extra_names = []
2530 annotation_names = []
2531 if not self.extra and not self.annotations:
2532 # Shortcut - if there are no extra or annotations, then
2533 # the values() clause must be just field names.
2534 field_names = list(fields)
2535 selected = dict(zip(fields, range(len(fields))))
2536 else:
2537 self.default_cols = False
2538 for f in fields:
2539 if extra := self.extra_select.get(f):
2540 extra_names.append(f)
2541 selected[f] = RawSQL(*extra)
2542 elif f in self.annotation_select:
2543 annotation_names.append(f)
2544 selected[f] = f
2545 elif f in self.annotations:
2546 raise FieldError(
2547 f"Cannot select the '{f}' alias. Use annotate() to "
2548 "promote it."
2549 )
2550 else:
2551 # Call `names_to_path` to ensure a FieldError including
2552 # annotations about to be masked as valid choices if
2553 # `f` is not resolvable.
2554 if self.annotation_select:
2555 self.names_to_path(f.split(LOOKUP_SEP), self.model._meta)
2556 selected[f] = len(field_names)
2557 field_names.append(f)
2558 self.set_extra_mask(extra_names)
2559 self.set_annotation_mask(annotation_names)
2560 else:
2561 field_names = [f.attname for f in self.model._meta.concrete_fields]
2562 selected = dict.fromkeys(field_names, None)
2563 # Selected annotations must be known before setting the GROUP BY
2564 # clause.
2565 if self.group_by is True:
2566 self.add_fields(
2567 (f.attname for f in self.model._meta.concrete_fields), False
2568 )
2569 # Disable GROUP BY aliases to avoid orphaning references to the
2570 # SELECT clause which is about to be cleared.
2571 self.set_group_by(allow_aliases=False)
2572 self.clear_select_fields()
2573 elif self.group_by:
2574 # Resolve GROUP BY annotation references if they are not part of
2575 # the selected fields anymore.
2576 group_by = []
2577 for expr in self.group_by:
2578 if isinstance(expr, Ref) and expr.refs not in selected:
2579 expr = self.annotations[expr.refs]
2580 group_by.append(expr)
2581 self.group_by = tuple(group_by)
2582
2583 self.values_select = tuple(field_names)
2584 self.add_fields(field_names, True)
2585 self.selected = selected if fields else None
2586
2587 @property
2588 def annotation_select(self):
2589 """
2590 Return the dictionary of aggregate columns that are not masked and
2591 should be used in the SELECT clause. Cache this result for performance.
2592 """
2593 if self._annotation_select_cache is not None:
2594 return self._annotation_select_cache
2595 elif not self.annotations:
2596 return {}
2597 elif self.annotation_select_mask is not None:
2598 self._annotation_select_cache = {
2599 k: v
2600 for k, v in self.annotations.items()
2601 if k in self.annotation_select_mask
2602 }
2603 return self._annotation_select_cache
2604 else:
2605 return self.annotations
2606
2607 @property
2608 def extra_select(self):
2609 if self._extra_select_cache is not None:
2610 return self._extra_select_cache
2611 if not self.extra:
2612 return {}
2613 elif self.extra_select_mask is not None:
2614 self._extra_select_cache = {
2615 k: v for k, v in self.extra.items() if k in self.extra_select_mask
2616 }
2617 return self._extra_select_cache
2618 else:
2619 return self.extra
2620
2621 def trim_start(self, names_with_path):
2622 """
2623 Trim joins from the start of the join path. The candidates for trim
2624 are the PathInfos in names_with_path structure that are m2m joins.
2625
2626 Also set the select column so the start matches the join.
2627
2628 This method is meant to be used for generating the subquery joins &
2629 cols in split_exclude().
2630
2631 Return a lookup usable for doing outerq.filter(lookup=self) and a
2632 boolean indicating if the joins in the prefix contain a LEFT OUTER join.
2633 _"""
2634 all_paths = []
2635 for _, paths in names_with_path:
2636 all_paths.extend(paths)
2637 contains_louter = False
2638 # Trim and operate only on tables that were generated for
2639 # the lookup part of the query. That is, avoid trimming
2640 # joins generated for F() expressions.
2641 lookup_tables = [
2642 t for t in self.alias_map if t in self._lookup_joins or t == self.base_table
2643 ]
2644 for trimmed_paths, path in enumerate(all_paths):
2645 if path.m2m:
2646 break
2647 if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
2648 contains_louter = True
2649 alias = lookup_tables[trimmed_paths]
2650 self.unref_alias(alias)
2651 # The path.join_field is a Rel, lets get the other side's field
2652 join_field = path.join_field.field
2653 # Build the filter prefix.
2654 paths_in_prefix = trimmed_paths
2655 trimmed_prefix = []
2656 for name, path in names_with_path:
2657 if paths_in_prefix - len(path) < 0:
2658 break
2659 trimmed_prefix.append(name)
2660 paths_in_prefix -= len(path)
2661 trimmed_prefix.append(join_field.foreign_related_fields[0].name)
2662 trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
2663 # Lets still see if we can trim the first join from the inner query
2664 # (that is, self). We can't do this for:
2665 # - LEFT JOINs because we would miss those rows that have nothing on
2666 # the outer side,
2667 # - INNER JOINs from filtered relations because we would miss their
2668 # filters.
2669 first_join = self.alias_map[lookup_tables[trimmed_paths + 1]]
2670 if first_join.join_type != LOUTER and not first_join.filtered_relation:
2671 select_fields = [r[0] for r in join_field.related_fields]
2672 select_alias = lookup_tables[trimmed_paths + 1]
2673 self.unref_alias(lookup_tables[trimmed_paths])
2674 extra_restriction = join_field.get_extra_restriction(
2675 None, lookup_tables[trimmed_paths + 1]
2676 )
2677 if extra_restriction:
2678 self.where.add(extra_restriction, AND)
2679 else:
2680 # TODO: It might be possible to trim more joins from the start of the
2681 # inner query if it happens to have a longer join chain containing the
2682 # values in select_fields. Lets punt this one for now.
2683 select_fields = [r[1] for r in join_field.related_fields]
2684 select_alias = lookup_tables[trimmed_paths]
2685 # The found starting point is likely a join_class instead of a
2686 # base_table_class reference. But the first entry in the query's FROM
2687 # clause must not be a JOIN.
2688 for table in self.alias_map:
2689 if self.alias_refcount[table] > 0:
2690 self.alias_map[table] = self.base_table_class(
2691 self.alias_map[table].table_name,
2692 table,
2693 )
2694 break
2695 self.set_select([f.get_col(select_alias) for f in select_fields])
2696 return trimmed_prefix, contains_louter
2697
2698 def is_nullable(self, field):
2699 """
2700 Check if the given field should be treated as nullable.
2701
2702 Some backends treat '' as null and Django treats such fields as
2703 nullable for those backends. In such situations field.null can be
2704 False even if we should treat the field as nullable.
2705 """
2706 # We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have
2707 # (nor should it have) knowledge of which connection is going to be
2708 # used. The proper fix would be to defer all decisions where
2709 # is_nullable() is needed to the compiler stage, but that is not easy
2710 # to do currently.
2711 return field.null or (
2712 field.empty_strings_allowed
2713 and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
2714 )
2715
2716
2717 def get_order_dir(field, default="ASC"):
2718 """
2719 Return the field name and direction for an order specification. For
2720 example, '-foo' is returned as ('foo', 'DESC').
2721
2722 The 'default' param is used to indicate which way no prefix (or a '+'
2723 prefix) should sort. The '-' prefix always sorts the opposite way.
2724 """
2725 dirn = ORDER_DIR[default]
2726 if field[0] == "-":
2727 return field[1:], dirn[1]
2728 return field, dirn[0]
2729
2730
2731 class JoinPromoter:
2732 """
2733 A class to abstract away join promotion problems for complex filter
2734 conditions.
2735 """
2736
2737 def __init__(self, connector, num_children, negated):
2738 self.connector = connector
2739 self.negated = negated
2740 if self.negated:
2741 if connector == AND:
2742 self.effective_connector = OR
2743 else:
2744 self.effective_connector = AND
2745 else:
2746 self.effective_connector = self.connector
2747 self.num_children = num_children
2748 # Maps of table alias to how many times it is seen as required for
2749 # inner and/or outer joins.
2750 self.votes = Counter()
2751
2752 def __repr__(self):
2753 return (
2754 f"{self.__class__.__qualname__}(connector={self.connector!r}, "
2755 f"num_children={self.num_children!r}, negated={self.negated!r})"
2756 )
2757
2758 def add_votes(self, votes):
2759 """
2760 Add single vote per item to self.votes. Parameter can be any
2761 iterable.
2762 """
2763 self.votes.update(votes)
2764
2765 def update_join_types(self, query):
2766 """
2767 Change join types so that the generated query is as efficient as
2768 possible, but still correct. So, change as many joins as possible
2769 to INNER, but don't make OUTER joins INNER if that could remove
2770 results from the query.
2771 """
2772 to_promote = set()
2773 to_demote = set()
2774 # The effective_connector is used so that NOT (a AND b) is treated
2775 # similarly to (a OR b) for join promotion.
2776 for table, votes in self.votes.items():
2777 # We must use outer joins in OR case when the join isn't contained
2778 # in all of the joins. Otherwise the INNER JOIN itself could remove
2779 # valid results. Consider the case where a model with rel_a and
2780 # rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now,
2781 # if rel_a join doesn't produce any results is null (for example
2782 # reverse foreign key or null value in direct foreign key), and
2783 # there is a matching row in rel_b with col=2, then an INNER join
2784 # to rel_a would remove a valid match from the query. So, we need
2785 # to promote any existing INNER to LOUTER (it is possible this
2786 # promotion in turn will be demoted later on).
2787 if self.effective_connector == OR and votes < self.num_children:
2788 to_promote.add(table)
2789 # If connector is AND and there is a filter that can match only
2790 # when there is a joinable row, then use INNER. For example, in
2791 # rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL
2792 # as join output, then the col=1 or col=2 can't match (as
2793 # NULL=anything is always false).
2794 # For the OR case, if all children voted for a join to be inner,
2795 # then we can use INNER for the join. For example:
2796 # (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell)
2797 # then if rel_a doesn't produce any rows, the whole condition
2798 # can't match. Hence we can safely use INNER join.
2799 if self.effective_connector == AND or (
2800 self.effective_connector == OR and votes == self.num_children
2801 ):
2802 to_demote.add(table)
2803 # Finally, what happens in cases where we have:
2804 # (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0
2805 # Now, we first generate the OR clause, and promote joins for it
2806 # in the first if branch above. Both rel_a and rel_b are promoted
2807 # to LOUTER joins. After that we do the AND case. The OR case
2808 # voted no inner joins but the rel_a__col__gte=0 votes inner join
2809 # for rel_a. We demote it back to INNER join (in AND case a single
2810 # vote is enough). The demotion is OK, if rel_a doesn't produce
2811 # rows, then the rel_a__col__gte=0 clause can't be true, and thus
2812 # the whole clause must be false. So, it is safe to use INNER
2813 # join.
2814 # Note that in this example we could just as well have the __gte
2815 # clause and the OR clause swapped. Or we could replace the __gte
2816 # clause with an OR clause containing rel_a__col=1|rel_a__col=2,
2817 # and again we could safely demote to INNER.
2818 query.promote_joins(to_promote)
2819 query.demote_joins(to_demote)
2820 return to_demote