Debian Codegraph @

django/db/models/sql/compiler.py

raw
1 import collections
2 import json
3 import re
4 from functools import partial
5 from itertools import chain
6
7 from django.core.exceptions import EmptyResultSet, FieldError, FullResultSet
8 from django.db import DatabaseError, NotSupportedError
9 from django.db.models.constants import LOOKUP_SEP
10 from django.db.models.expressions import ColPairs, F, OrderBy, RawSQL, Ref, Value
11 from django.db.models.fields import AutoField, composite
12 from django.db.models.functions import Cast, Random
13 from django.db.models.lookups import Lookup
14 from django.db.models.query_utils import select_related_descend
15 from django.db.models.sql.constants import (
16 CURSOR,
17 GET_ITERATOR_CHUNK_SIZE,
18 MULTI,
19 NO_RESULTS,
20 ORDER_DIR,
21 ROW_COUNT,
22 SINGLE,
23 )
24 from django.db.models.sql.query import Query, get_order_dir
25 from django.db.transaction import TransactionManagementError
26 from django.utils.functional import cached_property
27 from django.utils.hashable import make_hashable
28 from django.utils.regex_helper import _lazy_re_compile
29
30
31 class PositionRef(Ref):
32 def __init__(self, ordinal, refs, source):
33 self.ordinal = ordinal
34 super().__init__(refs, source)
35
36 def as_sql(self, compiler, connection):
37 return str(self.ordinal), ()
38
39
40 class SQLCompiler:
41 # Multiline ordering SQL clause may appear from RawSQL.
42 ordering_parts = _lazy_re_compile(
43 r"^(.*)\s(?:ASC|DESC).*",
44 re.MULTILINE | re.DOTALL,
45 )
46
47 def __init__(self, query, connection, using, elide_empty=True):
48 self.query = query
49 self.connection = connection
50 self.using = using
51 # Some queries, e.g. coalesced aggregation, need to be executed even if
52 # they would return an empty result set.
53 self.elide_empty = elide_empty
54 self.quote_cache = {"*": "*"}
55 # The select, klass_info, and annotations are needed by QuerySet.iterator()
56 # these are set as a side-effect of executing the query. Note that we calculate
57 # separately a list of extra select columns needed for grammatical correctness
58 # of the query, but these columns are not included in self.select.
59 self.select = None
60 self.annotation_col_map = None
61 self.klass_info = None
62 self._meta_ordering = None
63
64 def __repr__(self):
65 return (
66 f"<{self.__class__.__qualname__} "
67 f"model={self.query.model.__qualname__} "
68 f"connection={self.connection!r} using={self.using!r}>"
69 )
70
71 def setup_query(self, with_col_aliases=False):
72 if all(self.query.alias_refcount[a] == 0 for a in self.query.alias_map):
73 self.query.get_initial_alias()
74 self.select, self.klass_info, self.annotation_col_map = self.get_select(
75 with_col_aliases=with_col_aliases,
76 )
77 self.col_count = len(self.select)
78
79 def pre_sql_setup(self, with_col_aliases=False):
80 """
81 Do any necessary class setup immediately prior to producing SQL. This
82 is for things that can't necessarily be done in __init__ because we
83 might not have all the pieces in place at that time.
84 """
85 self.setup_query(with_col_aliases=with_col_aliases)
86 order_by = self.get_order_by()
87 self.where, self.having, self.qualify = self.query.where.split_having_qualify(
88 must_group_by=self.query.group_by is not None
89 )
90 extra_select = self.get_extra_select(order_by, self.select)
91 self.has_extra_select = bool(extra_select)
92 group_by = self.get_group_by(self.select + extra_select, order_by)
93 return extra_select, order_by, group_by
94
95 def get_group_by(self, select, order_by):
96 """
97 Return a list of 2-tuples of form (sql, params).
98
99 The logic of what exactly the GROUP BY clause contains is hard
100 to describe in other words than "if it passes the test suite,
101 then it is correct".
102 """
103 # Some examples:
104 # SomeModel.objects.annotate(Count('somecol'))
105 # GROUP BY: all fields of the model
106 #
107 # SomeModel.objects.values('name').annotate(Count('somecol'))
108 # GROUP BY: name
109 #
110 # SomeModel.objects.annotate(Count('somecol')).values('name')
111 # GROUP BY: all cols of the model
112 #
113 # SomeModel.objects.values('name', 'pk')
114 # .annotate(Count('somecol')).values('pk')
115 # GROUP BY: name, pk
116 #
117 # SomeModel.objects.values('name').annotate(Count('somecol')).values('pk')
118 # GROUP BY: name, pk
119 #
120 # In fact, the self.query.group_by is the minimal set to GROUP BY. It
121 # can't be ever restricted to a smaller set, but additional columns in
122 # HAVING, ORDER BY, and SELECT clauses are added to it. Unfortunately
123 # the end result is that it is impossible to force the query to have
124 # a chosen GROUP BY clause - you can almost do this by using the form:
125 # .values(*wanted_cols).annotate(AnAggregate())
126 # but any later annotations, extra selects, values calls that
127 # refer some column outside of the wanted_cols, order_by, or even
128 # filter calls can alter the GROUP BY clause.
129
130 # The query.group_by is either None (no GROUP BY at all), True
131 # (group by select fields), or a list of expressions to be added
132 # to the group by.
133 if self.query.group_by is None:
134 return []
135 expressions = []
136 group_by_refs = set()
137 if self.query.group_by is not True:
138 # If the group by is set to a list (by .values() call most likely),
139 # then we need to add everything in it to the GROUP BY clause.
140 # Backwards compatibility hack for setting query.group_by. Remove
141 # when we have public API way of forcing the GROUP BY clause.
142 # Converts string references to expressions.
143 for expr in self.query.group_by:
144 if not hasattr(expr, "as_sql"):
145 expr = self.query.resolve_ref(expr)
146 if isinstance(expr, Ref):
147 if expr.refs not in group_by_refs:
148 group_by_refs.add(expr.refs)
149 expressions.append(expr.source)
150 else:
151 expressions.append(expr)
152 # Note that even if the group_by is set, it is only the minimal
153 # set to group by. So, we need to add cols in select, order_by, and
154 # having into the select in any case.
155 selected_expr_positions = {}
156 for ordinal, (expr, _, alias) in enumerate(select, start=1):
157 if alias:
158 selected_expr_positions[expr] = ordinal
159 # Skip members of the select clause that are already explicitly
160 # grouped against.
161 if alias in group_by_refs:
162 continue
163 expressions.extend(expr.get_group_by_cols())
164 if not self._meta_ordering:
165 for expr, (sql, params, is_ref) in order_by:
166 # Skip references to the SELECT clause, as all expressions in
167 # the SELECT clause are already part of the GROUP BY.
168 if not is_ref:
169 expressions.extend(expr.get_group_by_cols())
170 having_group_by = self.having.get_group_by_cols() if self.having else ()
171 for expr in having_group_by:
172 expressions.append(expr)
173 result = []
174 seen = set()
175 expressions = self.collapse_group_by(expressions, having_group_by)
176
177 allows_group_by_select_index = (
178 self.connection.features.allows_group_by_select_index
179 )
180 for expr in expressions:
181 try:
182 sql, params = self.compile(expr)
183 except (EmptyResultSet, FullResultSet):
184 continue
185 if (
186 allows_group_by_select_index
187 and (position := selected_expr_positions.get(expr)) is not None
188 ):
189 sql, params = str(position), ()
190 else:
191 sql, params = expr.select_format(self, sql, params)
192 params_hash = make_hashable(params)
193 if (sql, params_hash) not in seen:
194 result.append((sql, params))
195 seen.add((sql, params_hash))
196 return result
197
198 def collapse_group_by(self, expressions, having):
199 # If the database supports group by functional dependence reduction,
200 # then the expressions can be reduced to the set of selected table
201 # primary keys as all other columns are functionally dependent on them.
202 if self.connection.features.allows_group_by_selected_pks:
203 # Filter out all expressions associated with a table's primary key
204 # present in the grouped columns. This is done by identifying all
205 # tables that have their primary key included in the grouped
206 # columns and removing non-primary key columns referring to them.
207 # Unmanaged models are excluded because they could be representing
208 # database views on which the optimization might not be allowed.
209 pks = {
210 expr
211 for expr in expressions
212 if (
213 hasattr(expr, "target")
214 and expr.target.primary_key
215 and self.connection.features.allows_group_by_selected_pks_on_model(
216 expr.target.model
217 )
218 )
219 }
220 aliases = {expr.alias for expr in pks}
221 expressions = [
222 expr
223 for expr in expressions
224 if expr in pks
225 or expr in having
226 or getattr(expr, "alias", None) not in aliases
227 ]
228 return expressions
229
230 def get_select(self, with_col_aliases=False):
231 """
232 Return three values:
233 - a list of 3-tuples of (expression, (sql, params), alias)
234 - a klass_info structure,
235 - a dictionary of annotations
236
237 The (sql, params) is what the expression will produce, and alias is the
238 "AS alias" for the column (possibly None).
239
240 The klass_info structure contains the following information:
241 - The base model of the query.
242 - Which columns for that model are present in the query (by
243 position of the select clause).
244 - related_klass_infos: [f, klass_info] to descent into
245
246 The annotations is a dictionary of {'attname': column position} values.
247 """
248 select = []
249 klass_info = None
250 annotations = {}
251 assert not (self.query.select and self.query.default_cols)
252 select_mask = self.query.get_select_mask()
253 if self.query.default_cols:
254 cols = self.get_default_columns(select_mask)
255 else:
256 # self.query.select is a special case. These columns never go to
257 # any model.
258 cols = self.query.select
259 selected = []
260 select_fields = None
261 if self.query.selected is None:
262 selected = [
263 *(
264 (alias, RawSQL(*args))
265 for alias, args in self.query.extra_select.items()
266 ),
267 *((None, col) for col in cols),
268 *self.query.annotation_select.items(),
269 ]
270 select_fields = list(
271 range(
272 len(self.query.extra_select),
273 len(self.query.extra_select) + len(cols),
274 )
275 )
276 else:
277 select_fields = []
278 for index, (alias, expression) in enumerate(self.query.selected.items()):
279 # Reference to an annotation.
280 if isinstance(expression, str):
281 expression = self.query.annotations[expression]
282 # Reference to a column.
283 elif isinstance(expression, int):
284 select_fields.append(index)
285 expression = cols[expression]
286 # ColPairs cannot be aliased.
287 if isinstance(expression, ColPairs):
288 alias = None
289 selected.append((alias, expression))
290 if select_fields:
291 klass_info = {"model": self.query.model, "select_fields": select_fields}
292
293 for select_idx, (alias, expression) in enumerate(selected):
294 if alias:
295 annotations[alias] = select_idx
296 select.append((expression, alias))
297
298 if self.query.select_related:
299 related_klass_infos = self.get_related_selections(select, select_mask)
300 klass_info["related_klass_infos"] = related_klass_infos
301
302 def get_select_from_parent(klass_info):
303 for ki in klass_info["related_klass_infos"]:
304 if ki["from_parent"]:
305 ki["select_fields"] = (
306 klass_info["select_fields"] + ki["select_fields"]
307 )
308 get_select_from_parent(ki)
309
310 get_select_from_parent(klass_info)
311
312 ret = []
313 col_idx = 1
314 for col, alias in select:
315 try:
316 sql, params = self.compile(col)
317 except EmptyResultSet:
318 empty_result_set_value = getattr(
319 col, "empty_result_set_value", NotImplemented
320 )
321 if empty_result_set_value is NotImplemented:
322 # Select a predicate that's always False.
323 sql, params = "0", ()
324 else:
325 sql, params = self.compile(Value(empty_result_set_value))
326 except FullResultSet:
327 sql, params = self.compile(Value(True))
328 else:
329 sql, params = col.select_format(self, sql, params)
330 if alias is None and with_col_aliases:
331 alias = f"col{col_idx}"
332 col_idx += 1
333 ret.append((col, (sql, params), alias))
334 return ret, klass_info, annotations
335
336 def _order_by_pairs(self):
337 if self.query.extra_order_by:
338 ordering = self.query.extra_order_by
339 elif not self.query.default_ordering:
340 ordering = self.query.order_by
341 elif self.query.order_by:
342 ordering = self.query.order_by
343 elif (meta := self.query.get_meta()) and meta.ordering:
344 ordering = meta.ordering
345 self._meta_ordering = ordering
346 else:
347 ordering = []
348 if self.query.standard_ordering:
349 default_order, _ = ORDER_DIR["ASC"]
350 else:
351 default_order, _ = ORDER_DIR["DESC"]
352
353 selected_exprs = {}
354 # Avoid computing `selected_exprs` if there is no `ordering` as it's
355 # relatively expensive.
356 if ordering and (select := self.select):
357 for ordinal, (expr, _, alias) in enumerate(select, start=1):
358 pos_expr = PositionRef(ordinal, alias, expr)
359 if alias:
360 selected_exprs[alias] = pos_expr
361 selected_exprs[expr] = pos_expr
362
363 for field in ordering:
364 if hasattr(field, "resolve_expression"):
365 if isinstance(field, Value):
366 # output_field must be resolved for constants.
367 field = Cast(field, field.output_field)
368 if not isinstance(field, OrderBy):
369 field = field.asc()
370 if not self.query.standard_ordering:
371 field = field.copy()
372 field.reverse_ordering()
373 select_ref = selected_exprs.get(field.expression)
374 if select_ref or (
375 isinstance(field.expression, F)
376 and (select_ref := selected_exprs.get(field.expression.name))
377 ):
378 # Emulation of NULLS (FIRST|LAST) cannot be combined with
379 # the usage of ordering by position.
380 if (
381 field.nulls_first is None and field.nulls_last is None
382 ) or self.connection.features.supports_order_by_nulls_modifier:
383 field = field.copy()
384 field.expression = select_ref
385 # Alias collisions are not possible when dealing with
386 # combined queries so fallback to it if emulation of NULLS
387 # handling is required.
388 elif self.query.combinator:
389 field = field.copy()
390 field.expression = Ref(select_ref.refs, select_ref.source)
391 yield field, select_ref is not None
392 continue
393 if field == "?": # random
394 yield OrderBy(Random()), False
395 continue
396
397 col, order = get_order_dir(field, default_order)
398 descending = order == "DESC"
399
400 if select_ref := selected_exprs.get(col):
401 # Reference to expression in SELECT clause
402 yield (
403 OrderBy(
404 select_ref,
405 descending=descending,
406 ),
407 True,
408 )
409 continue
410
411 if expr := self.query.annotations.get(col):
412 ref = col
413 transforms = []
414 else:
415 ref, *transforms = col.split(LOOKUP_SEP)
416 expr = self.query.annotations.get(ref)
417 if expr:
418 if self.query.combinator and self.select:
419 if transforms:
420 raise NotImplementedError(
421 "Ordering combined queries by transforms is not "
422 "implemented."
423 )
424 # Don't use the resolved annotation because other
425 # combined queries might define it differently.
426 expr = F(ref)
427 if transforms:
428 for name in transforms:
429 expr = self.query.try_transform(expr, name)
430 if isinstance(expr, Value):
431 # output_field must be resolved for constants.
432 expr = Cast(expr, expr.output_field)
433 yield OrderBy(expr, descending=descending), False
434 continue
435
436 if "." in field and field in self.query.extra_order_by:
437 # This came in through an extra(order_by=...) addition. Pass it
438 # on verbatim.
439 table, col = col.split(".", 1)
440 yield (
441 OrderBy(
442 RawSQL(
443 "%s.%s" % (self.quote_name_unless_alias(table), col), []
444 ),
445 descending=descending,
446 ),
447 False,
448 )
449 continue
450
451 if self.query.extra and col in self.query.extra:
452 if col in self.query.extra_select:
453 yield (
454 OrderBy(
455 Ref(col, RawSQL(*self.query.extra[col])),
456 descending=descending,
457 ),
458 True,
459 )
460 else:
461 yield (
462 OrderBy(RawSQL(*self.query.extra[col]), descending=descending),
463 False,
464 )
465 else:
466 if self.query.combinator and self.select:
467 # Don't use the first model's field because other
468 # combinated queries might define it differently.
469 yield OrderBy(F(col), descending=descending), False
470 else:
471 # 'col' is of the form 'field' or 'field1__field2' or
472 # '-field1__field2__field', etc.
473 yield from self.find_ordering_name(
474 field,
475 self.query.get_meta(),
476 default_order=default_order,
477 )
478
479 def get_order_by(self):
480 """
481 Return a list of 2-tuples of the form (expr, (sql, params, is_ref)) for
482 the ORDER BY clause.
483
484 The order_by clause can alter the select clause (for example it can add
485 aliases to clauses that do not yet have one, or it can add totally new
486 select clauses).
487 """
488 result = []
489 seen = set()
490 for expr, is_ref in self._order_by_pairs():
491 resolved = expr.resolve_expression(self.query, allow_joins=True, reuse=None)
492 if not is_ref and self.query.combinator and self.select:
493 src = resolved.expression
494 expr_src = expr.expression
495 for sel_expr, _, col_alias in self.select:
496 if src == sel_expr:
497 # When values() is used the exact alias must be used to
498 # reference annotations.
499 if (
500 self.query.has_select_fields
501 and col_alias in self.query.annotation_select
502 and not (
503 isinstance(expr_src, F) and col_alias == expr_src.name
504 )
505 ):
506 continue
507 resolved.set_source_expressions(
508 [Ref(col_alias if col_alias else src.target.column, src)]
509 )
510 break
511 else:
512 # Add column used in ORDER BY clause to the selected
513 # columns and to each combined query.
514 order_by_idx = len(self.query.select) + 1
515 col_alias = f"__orderbycol{order_by_idx}"
516 for q in self.query.combined_queries:
517 # If fields were explicitly selected through values()
518 # combined queries cannot be augmented.
519 if q.has_select_fields:
520 raise DatabaseError(
521 "ORDER BY term does not match any column in "
522 "the result set."
523 )
524 q.add_annotation(expr_src, col_alias)
525 self.query.add_select_col(resolved, col_alias)
526 resolved.set_source_expressions([Ref(col_alias, src)])
527 sql, params = self.compile(resolved)
528 # Don't add the same column twice, but the order direction is
529 # not taken into account so we strip it. When this entire method
530 # is refactored into expressions, then we can check each part as we
531 # generate it.
532 without_ordering = self.ordering_parts.search(sql)[1]
533 params_hash = make_hashable(params)
534 if (without_ordering, params_hash) in seen:
535 continue
536 seen.add((without_ordering, params_hash))
537 result.append((resolved, (sql, params, is_ref)))
538 return result
539
540 def get_extra_select(self, order_by, select):
541 extra_select = []
542 if self.query.distinct and not self.query.distinct_fields:
543 select_sql = [t[1] for t in select]
544 for expr, (sql, params, is_ref) in order_by:
545 without_ordering = self.ordering_parts.search(sql)[1]
546 if not is_ref and (without_ordering, params) not in select_sql:
547 extra_select.append((expr, (without_ordering, params), None))
548 return extra_select
549
550 def quote_name_unless_alias(self, name):
551 """
552 A wrapper around connection.ops.quote_name that doesn't quote aliases
553 for table names. This avoids problems with some SQL dialects that treat
554 quoted strings specially (e.g. PostgreSQL).
555 """
556 if name in self.quote_cache:
557 return self.quote_cache[name]
558 if (
559 (name in self.query.alias_map and name not in self.query.table_map)
560 or name in self.query.extra_select
561 or (
562 self.query.external_aliases.get(name)
563 and name not in self.query.table_map
564 )
565 ):
566 self.quote_cache[name] = name
567 return name
568 r = self.connection.ops.quote_name(name)
569 self.quote_cache[name] = r
570 return r
571
572 def compile(self, node):
573 vendor_impl = getattr(node, "as_" + self.connection.vendor, None)
574 if vendor_impl:
575 sql, params = vendor_impl(self, self.connection)
576 else:
577 sql, params = node.as_sql(self, self.connection)
578 return sql, params
579
580 def get_combinator_sql(self, combinator, all):
581 features = self.connection.features
582 compilers = [
583 query.get_compiler(self.using, self.connection, self.elide_empty)
584 for query in self.query.combined_queries
585 ]
586 if not features.supports_slicing_ordering_in_compound:
587 for compiler in compilers:
588 if compiler.query.is_sliced:
589 raise DatabaseError(
590 "LIMIT/OFFSET not allowed in subqueries of compound statements."
591 )
592 if compiler.get_order_by():
593 raise DatabaseError(
594 "ORDER BY not allowed in subqueries of compound statements."
595 )
596 parts = []
597 empty_compiler = None
598 for compiler in compilers:
599 try:
600 parts.append(self._get_combinator_part_sql(compiler))
601 except EmptyResultSet:
602 # Omit the empty queryset with UNION and with DIFFERENCE if the
603 # first queryset is nonempty.
604 if combinator == "union" or (combinator == "difference" and parts):
605 empty_compiler = compiler
606 continue
607 raise
608 if not parts:
609 raise EmptyResultSet
610 elif len(parts) == 1 and combinator == "union" and self.query.is_sliced:
611 # A sliced union cannot be composed of a single component because
612 # in the event the later is also sliced it might result in invalid
613 # SQL due to the usage of multiple LIMIT clauses. Prevent that from
614 # happening by always including an empty resultset query to force
615 # the creation of an union.
616 empty_compiler.elide_empty = False
617 parts.append(self._get_combinator_part_sql(empty_compiler))
618 combinator_sql = self.connection.ops.set_operators[combinator]
619 if all and combinator == "union":
620 combinator_sql += " ALL"
621 braces = "{}"
622 if not self.query.subquery and features.supports_slicing_ordering_in_compound:
623 braces = "({})"
624 sql_parts, args_parts = zip(
625 *((braces.format(sql), args) for sql, args in parts)
626 )
627 result = [" {} ".format(combinator_sql).join(sql_parts)]
628 params = []
629 for part in args_parts:
630 params.extend(part)
631 return result, params
632
633 def _get_combinator_part_sql(self, compiler):
634 features = self.connection.features
635 # If the columns list is limited, then all combined queries
636 # must have the same columns list. Set the selects defined on
637 # the query on all combined queries, if not already set.
638 selected = self.query.selected
639 if selected is not None and compiler.query.selected is None:
640 compiler.query = compiler.query.clone()
641 compiler.query.set_values(selected)
642 part_sql, part_args = compiler.as_sql(with_col_aliases=True)
643 if compiler.query.combinator:
644 # Wrap in a subquery if wrapping in parentheses isn't
645 # supported.
646 if not features.supports_parentheses_in_compound:
647 part_sql = "SELECT * FROM ({})".format(part_sql)
648 # Add parentheses when combining with compound query if not
649 # already added for all compound queries.
650 elif (
651 self.query.subquery
652 or not features.supports_slicing_ordering_in_compound
653 ):
654 part_sql = "({})".format(part_sql)
655 elif self.query.subquery and features.supports_slicing_ordering_in_compound:
656 part_sql = "({})".format(part_sql)
657 return part_sql, part_args
658
659 def get_qualify_sql(self):
660 where_parts = []
661 if self.where:
662 where_parts.append(self.where)
663 if self.having:
664 where_parts.append(self.having)
665 inner_query = self.query.clone()
666 inner_query.subquery = True
667 inner_query.where = inner_query.where.__class__(where_parts)
668 # Augment the inner query with any window function references that
669 # might have been masked via values() and alias(). If any masked
670 # aliases are added they'll be masked again to avoid fetching
671 # the data in the `if qual_aliases` branch below.
672 select = {
673 expr: alias for expr, _, alias in self.get_select(with_col_aliases=True)[0]
674 }
675 select_aliases = set(select.values())
676 qual_aliases = set()
677 replacements = {}
678
679 def collect_replacements(expressions):
680 while expressions:
681 expr = expressions.pop()
682 if expr in replacements:
683 continue
684 elif select_alias := select.get(expr):
685 replacements[expr] = select_alias
686 elif isinstance(expr, Lookup):
687 expressions.extend(expr.get_source_expressions())
688 elif isinstance(expr, Ref):
689 if expr.refs not in select_aliases:
690 expressions.extend(expr.get_source_expressions())
691 else:
692 num_qual_alias = len(qual_aliases)
693 select_alias = f"qual{num_qual_alias}"
694 qual_aliases.add(select_alias)
695 inner_query.add_annotation(expr, select_alias)
696 replacements[expr] = select_alias
697
698 collect_replacements(list(self.qualify.leaves()))
699 self.qualify = self.qualify.replace_expressions(
700 {expr: Ref(alias, expr) for expr, alias in replacements.items()}
701 )
702 order_by = []
703 for order_by_expr, *_ in self.get_order_by():
704 collect_replacements(order_by_expr.get_source_expressions())
705 order_by.append(
706 order_by_expr.replace_expressions(
707 {expr: Ref(alias, expr) for expr, alias in replacements.items()}
708 )
709 )
710 inner_query_compiler = inner_query.get_compiler(
711 self.using, connection=self.connection, elide_empty=self.elide_empty
712 )
713 inner_sql, inner_params = inner_query_compiler.as_sql(
714 # The limits must be applied to the outer query to avoid pruning
715 # results too eagerly.
716 with_limits=False,
717 # Force unique aliasing of selected columns to avoid collisions
718 # and make rhs predicates referencing easier.
719 with_col_aliases=True,
720 )
721 qualify_sql, qualify_params = self.compile(self.qualify)
722 result = [
723 "SELECT * FROM (",
724 inner_sql,
725 ")",
726 self.connection.ops.quote_name("qualify"),
727 "WHERE",
728 qualify_sql,
729 ]
730 if qual_aliases:
731 # If some select aliases were unmasked for filtering purposes they
732 # must be masked back.
733 cols = [self.connection.ops.quote_name(alias) for alias in select.values()]
734 result = [
735 "SELECT",
736 ", ".join(cols),
737 "FROM (",
738 *result,
739 ")",
740 self.connection.ops.quote_name("qualify_mask"),
741 ]
742 params = list(inner_params) + qualify_params
743 # As the SQL spec is unclear on whether or not derived tables
744 # ordering must propagate it has to be explicitly repeated on the
745 # outer-most query to ensure it's preserved.
746 if order_by:
747 ordering_sqls = []
748 for ordering in order_by:
749 ordering_sql, ordering_params = self.compile(ordering)
750 ordering_sqls.append(ordering_sql)
751 params.extend(ordering_params)
752 result.extend(["ORDER BY", ", ".join(ordering_sqls)])
753 return result, params
754
755 def as_sql(self, with_limits=True, with_col_aliases=False):
756 """
757 Create the SQL for this query. Return the SQL string and list of
758 parameters.
759
760 If 'with_limits' is False, any limit/offset information is not included
761 in the query.
762 """
763 refcounts_before = self.query.alias_refcount.copy()
764 try:
765 combinator = self.query.combinator
766 extra_select, order_by, group_by = self.pre_sql_setup(
767 with_col_aliases=with_col_aliases or bool(combinator),
768 )
769 for_update_part = None
770 # Is a LIMIT/OFFSET clause needed?
771 with_limit_offset = with_limits and self.query.is_sliced
772 combinator = self.query.combinator
773 features = self.connection.features
774 if combinator:
775 if not getattr(features, "supports_select_{}".format(combinator)):
776 raise NotSupportedError(
777 "{} is not supported on this database backend.".format(
778 combinator
779 )
780 )
781 result, params = self.get_combinator_sql(
782 combinator, self.query.combinator_all
783 )
784 elif self.qualify:
785 result, params = self.get_qualify_sql()
786 order_by = None
787 else:
788 distinct_fields, distinct_params = self.get_distinct()
789 # This must come after 'select', 'ordering', and 'distinct'
790 # (see docstring of get_from_clause() for details).
791 from_, f_params = self.get_from_clause()
792 try:
793 where, w_params = (
794 self.compile(self.where) if self.where is not None else ("", [])
795 )
796 except EmptyResultSet:
797 if self.elide_empty:
798 raise
799 # Use a predicate that's always False.
800 where, w_params = "0 = 1", []
801 except FullResultSet:
802 where, w_params = "", []
803 try:
804 having, h_params = (
805 self.compile(self.having)
806 if self.having is not None
807 else ("", [])
808 )
809 except FullResultSet:
810 having, h_params = "", []
811 result = ["SELECT"]
812 params = []
813
814 if self.query.distinct:
815 distinct_result, distinct_params = self.connection.ops.distinct_sql(
816 distinct_fields,
817 distinct_params,
818 )
819 result += distinct_result
820 params += distinct_params
821
822 out_cols = []
823 for _, (s_sql, s_params), alias in self.select + extra_select:
824 if alias:
825 s_sql = "%s AS %s" % (
826 s_sql,
827 self.connection.ops.quote_name(alias),
828 )
829 params.extend(s_params)
830 out_cols.append(s_sql)
831
832 result += [", ".join(out_cols)]
833 if from_:
834 result += ["FROM", *from_]
835 elif self.connection.features.bare_select_suffix:
836 result += [self.connection.features.bare_select_suffix]
837 params.extend(f_params)
838
839 if self.query.select_for_update and features.has_select_for_update:
840 if (
841 self.connection.get_autocommit()
842 # Don't raise an exception when database doesn't
843 # support transactions, as it's a noop.
844 and features.supports_transactions
845 ):
846 raise TransactionManagementError(
847 "select_for_update cannot be used outside of a transaction."
848 )
849
850 if (
851 with_limit_offset
852 and not features.supports_select_for_update_with_limit
853 ):
854 raise NotSupportedError(
855 "LIMIT/OFFSET is not supported with "
856 "select_for_update on this database backend."
857 )
858 nowait = self.query.select_for_update_nowait
859 skip_locked = self.query.select_for_update_skip_locked
860 of = self.query.select_for_update_of
861 no_key = self.query.select_for_no_key_update
862 # If it's a NOWAIT/SKIP LOCKED/OF/NO KEY query but the
863 # backend doesn't support it, raise NotSupportedError to
864 # prevent a possible deadlock.
865 if nowait and not features.has_select_for_update_nowait:
866 raise NotSupportedError(
867 "NOWAIT is not supported on this database backend."
868 )
869 elif skip_locked and not features.has_select_for_update_skip_locked:
870 raise NotSupportedError(
871 "SKIP LOCKED is not supported on this database backend."
872 )
873 elif of and not features.has_select_for_update_of:
874 raise NotSupportedError(
875 "FOR UPDATE OF is not supported on this database backend."
876 )
877 elif no_key and not features.has_select_for_no_key_update:
878 raise NotSupportedError(
879 "FOR NO KEY UPDATE is not supported on this "
880 "database backend."
881 )
882 for_update_part = self.connection.ops.for_update_sql(
883 nowait=nowait,
884 skip_locked=skip_locked,
885 of=self.get_select_for_update_of_arguments(),
886 no_key=no_key,
887 )
888
889 if for_update_part and features.for_update_after_from:
890 result.append(for_update_part)
891
892 if where:
893 result.append("WHERE %s" % where)
894 params.extend(w_params)
895
896 grouping = []
897 for g_sql, g_params in group_by:
898 grouping.append(g_sql)
899 params.extend(g_params)
900 if grouping:
901 if distinct_fields:
902 raise NotImplementedError(
903 "annotate() + distinct(fields) is not implemented."
904 )
905 order_by = order_by or self.connection.ops.force_no_ordering()
906 result.append("GROUP BY %s" % ", ".join(grouping))
907 if self._meta_ordering:
908 order_by = None
909 if having:
910 if not grouping:
911 result.extend(self.connection.ops.force_group_by())
912 result.append("HAVING %s" % having)
913 params.extend(h_params)
914
915 if self.query.explain_info:
916 result.insert(
917 0,
918 self.connection.ops.explain_query_prefix(
919 self.query.explain_info.format,
920 **self.query.explain_info.options,
921 ),
922 )
923
924 if order_by:
925 ordering = []
926 for _, (o_sql, o_params, _) in order_by:
927 ordering.append(o_sql)
928 params.extend(o_params)
929 order_by_sql = "ORDER BY %s" % ", ".join(ordering)
930 if combinator and features.requires_compound_order_by_subquery:
931 result = ["SELECT * FROM (", *result, ")", order_by_sql]
932 else:
933 result.append(order_by_sql)
934
935 if with_limit_offset:
936 result.append(
937 self.connection.ops.limit_offset_sql(
938 self.query.low_mark, self.query.high_mark
939 )
940 )
941
942 if for_update_part and not features.for_update_after_from:
943 result.append(for_update_part)
944
945 if self.query.subquery and extra_select:
946 # If the query is used as a subquery, the extra selects would
947 # result in more columns than the left-hand side expression is
948 # expecting. This can happen when a subquery uses a combination
949 # of order_by() and distinct(), forcing the ordering expressions
950 # to be selected as well. Wrap the query in another subquery
951 # to exclude extraneous selects.
952 sub_selects = []
953 sub_params = []
954 for index, (select, _, alias) in enumerate(self.select, start=1):
955 if alias:
956 sub_selects.append(
957 "%s.%s"
958 % (
959 self.connection.ops.quote_name("subquery"),
960 self.connection.ops.quote_name(alias),
961 )
962 )
963 else:
964 select_clone = select.relabeled_clone(
965 {select.alias: "subquery"}
966 )
967 subselect, subparams = select_clone.as_sql(
968 self, self.connection
969 )
970 sub_selects.append(subselect)
971 sub_params.extend(subparams)
972 return "SELECT %s FROM (%s) subquery" % (
973 ", ".join(sub_selects),
974 " ".join(result),
975 ), tuple(sub_params + params)
976
977 return " ".join(result), tuple(params)
978 finally:
979 # Finally do cleanup - get rid of the joins we created above.
980 self.query.reset_refcounts(refcounts_before)
981
982 def get_default_columns(
983 self, select_mask, start_alias=None, opts=None, from_parent=None
984 ):
985 """
986 Compute the default columns for selecting every field in the base
987 model. Will sometimes be called to pull in related models (e.g. via
988 select_related), in which case "opts" and "start_alias" will be given
989 to provide a starting point for the traversal.
990
991 Return a list of strings, quoted appropriately for use in SQL
992 directly, as well as a set of aliases used in the select statement (if
993 'as_pairs' is True, return a list of (alias, col_name) pairs instead
994 of strings as the first component and None as the second component).
995 """
996 result = []
997 if opts is None:
998 if (opts := self.query.get_meta()) is None:
999 return result
1000 start_alias = start_alias or self.query.get_initial_alias()
1001 # The 'seen_models' is used to optimize checking the needed parent
1002 # alias for a given field. This also includes None -> start_alias to
1003 # be used by local fields.
1004 seen_models = {None: start_alias}
1005 select_mask_fields = set(composite.unnest(select_mask))
1006
1007 for field in opts.concrete_fields:
1008 model = field.model._meta.concrete_model
1009 # A proxy model will have a different model and concrete_model. We
1010 # will assign None if the field belongs to this model.
1011 if model == opts.model:
1012 model = None
1013 if (
1014 from_parent
1015 and model is not None
1016 and issubclass(
1017 from_parent._meta.concrete_model, model._meta.concrete_model
1018 )
1019 ):
1020 # Avoid loading data for already loaded parents.
1021 # We end up here in the case select_related() resolution
1022 # proceeds from parent model to child model. In that case the
1023 # parent model data is already present in the SELECT clause,
1024 # and we want to avoid reloading the same data again.
1025 continue
1026 if select_mask and field not in select_mask_fields:
1027 continue
1028 alias = self.query.join_parent_model(opts, model, start_alias, seen_models)
1029 column = field.get_col(alias)
1030 result.append(column)
1031 return result
1032
1033 def get_distinct(self):
1034 """
1035 Return a quoted list of fields to use in DISTINCT ON part of the query.
1036
1037 This method can alter the tables in the query, and thus it must be
1038 called before get_from_clause().
1039 """
1040 result = []
1041 params = []
1042 opts = self.query.get_meta()
1043
1044 for name in self.query.distinct_fields:
1045 parts = name.split(LOOKUP_SEP)
1046 _, targets, alias, joins, path, _, transform_function = self._setup_joins(
1047 parts, opts, None
1048 )
1049 targets, alias, _ = self.query.trim_joins(targets, joins, path)
1050 for target in targets:
1051 if name in self.query.annotation_select:
1052 result.append(self.connection.ops.quote_name(name))
1053 else:
1054 r, p = self.compile(transform_function(target, alias))
1055 result.append(r)
1056 params.append(p)
1057 return result, params
1058
1059 def find_ordering_name(
1060 self, name, opts, alias=None, default_order="ASC", already_seen=None
1061 ):
1062 """
1063 Return the table alias (the name might be ambiguous, the alias will
1064 not be) and column name for ordering by the given 'name' parameter.
1065 The 'name' is of the form 'field1__field2__...__fieldN'.
1066 """
1067 name, order = get_order_dir(name, default_order)
1068 descending = order == "DESC"
1069 pieces = name.split(LOOKUP_SEP)
1070 (
1071 field,
1072 targets,
1073 alias,
1074 joins,
1075 path,
1076 opts,
1077 transform_function,
1078 ) = self._setup_joins(pieces, opts, alias)
1079
1080 # If we get to this point and the field is a relation to another model,
1081 # append the default ordering for that model unless it is the pk
1082 # shortcut or the attribute name of the field that is specified or
1083 # there are transforms to process.
1084 if (
1085 field.is_relation
1086 and opts.ordering
1087 and getattr(field, "attname", None) != pieces[-1]
1088 and name != "pk"
1089 and not getattr(transform_function, "has_transforms", False)
1090 ):
1091 # Firstly, avoid infinite loops.
1092 already_seen = already_seen or set()
1093 join_tuple = tuple(
1094 getattr(self.query.alias_map[j], "join_cols", None) for j in joins
1095 )
1096 if join_tuple in already_seen:
1097 raise FieldError("Infinite loop caused by ordering.")
1098 already_seen.add(join_tuple)
1099
1100 results = []
1101 for item in opts.ordering:
1102 if hasattr(item, "resolve_expression") and not isinstance(
1103 item, OrderBy
1104 ):
1105 item = item.desc() if descending else item.asc()
1106 if isinstance(item, OrderBy):
1107 results.append(
1108 (item.prefix_references(f"{name}{LOOKUP_SEP}"), False)
1109 )
1110 continue
1111 results.extend(
1112 (expr.prefix_references(f"{name}{LOOKUP_SEP}"), is_ref)
1113 for expr, is_ref in self.find_ordering_name(
1114 item, opts, alias, order, already_seen
1115 )
1116 )
1117 return results
1118 targets, alias, _ = self.query.trim_joins(targets, joins, path)
1119 return [
1120 (OrderBy(transform_function(t, alias), descending=descending), False)
1121 for t in targets
1122 ]
1123
1124 def _setup_joins(self, pieces, opts, alias):
1125 """
1126 Helper method for get_order_by() and get_distinct().
1127
1128 get_ordering() and get_distinct() must produce same target columns on
1129 same input, as the prefixes of get_ordering() and get_distinct() must
1130 match. Executing SQL where this is not true is an error.
1131 """
1132 alias = alias or self.query.get_initial_alias()
1133 field, targets, opts, joins, path, transform_function = self.query.setup_joins(
1134 pieces, opts, alias
1135 )
1136 alias = joins[-1]
1137 return field, targets, alias, joins, path, opts, transform_function
1138
1139 def get_from_clause(self):
1140 """
1141 Return a list of strings that are joined together to go after the
1142 "FROM" part of the query, as well as a list any extra parameters that
1143 need to be included. Subclasses, can override this to create a
1144 from-clause via a "select".
1145
1146 This should only be called after any SQL construction methods that
1147 might change the tables that are needed. This means the select columns,
1148 ordering, and distinct must be done first.
1149 """
1150 result = []
1151 params = []
1152 # Copy alias_map to a tuple in case Join.as_sql() subclasses (objects
1153 # in alias_map) alter compiler.query.alias_map. That would otherwise
1154 # raise "RuntimeError: dictionary changed size during iteration".
1155 for alias, from_clause in tuple(self.query.alias_map.items()):
1156 if not self.query.alias_refcount[alias]:
1157 continue
1158 clause_sql, clause_params = self.compile(from_clause)
1159 result.append(clause_sql)
1160 params.extend(clause_params)
1161 for t in self.query.extra_tables:
1162 alias, _ = self.query.table_alias(t)
1163 # Only add the alias if it's not already present (the table_alias()
1164 # call increments the refcount, so an alias refcount of one means
1165 # this is the only reference).
1166 if (
1167 alias not in self.query.alias_map
1168 or self.query.alias_refcount[alias] == 1
1169 ):
1170 result.append(", %s" % self.quote_name_unless_alias(alias))
1171 return result, params
1172
1173 def get_related_selections(
1174 self,
1175 select,
1176 select_mask,
1177 opts=None,
1178 root_alias=None,
1179 cur_depth=1,
1180 requested=None,
1181 restricted=None,
1182 ):
1183 """
1184 Fill in the information needed for a select_related query. The current
1185 depth is measured as the number of connections away from the root model
1186 (for example, cur_depth=1 means we are looking at models with direct
1187 connections to the root model).
1188 """
1189
1190 def _get_field_choices():
1191 direct_choices = (f.name for f in opts.fields if f.is_relation)
1192 reverse_choices = (
1193 f.field.related_query_name()
1194 for f in opts.related_objects
1195 if f.field.unique
1196 )
1197 return chain(
1198 direct_choices, reverse_choices, self.query._filtered_relations
1199 )
1200
1201 related_klass_infos = []
1202 if not restricted and cur_depth > self.query.max_depth:
1203 # We've recursed far enough; bail out.
1204 return related_klass_infos
1205
1206 if not opts:
1207 opts = self.query.get_meta()
1208 root_alias = self.query.get_initial_alias()
1209
1210 # Setup for the case when only particular related fields should be
1211 # included in the related selection.
1212 fields_found = set()
1213 if requested is None:
1214 restricted = isinstance(self.query.select_related, dict)
1215 if restricted:
1216 requested = self.query.select_related
1217
1218 def get_related_klass_infos(klass_info, related_klass_infos):
1219 klass_info["related_klass_infos"] = related_klass_infos
1220
1221 for f in opts.fields:
1222 fields_found.add(f.name)
1223
1224 if restricted:
1225 next = requested.get(f.name, {})
1226 if not f.is_relation:
1227 # If a non-related field is used like a relation,
1228 # or if a single non-relational field is given.
1229 if next or f.name in requested:
1230 raise FieldError(
1231 "Non-relational field given in select_related: '%s'. "
1232 "Choices are: %s"
1233 % (
1234 f.name,
1235 ", ".join(_get_field_choices()) or "(none)",
1236 )
1237 )
1238 else:
1239 next = False
1240
1241 if not select_related_descend(f, restricted, requested, select_mask):
1242 continue
1243 related_select_mask = select_mask.get(f) or {}
1244 klass_info = {
1245 "model": f.remote_field.model,
1246 "field": f,
1247 "reverse": False,
1248 "local_setter": f.set_cached_value,
1249 "remote_setter": (
1250 f.remote_field.set_cached_value if f.unique else lambda x, y: None
1251 ),
1252 "from_parent": False,
1253 }
1254 related_klass_infos.append(klass_info)
1255 select_fields = []
1256 _, _, _, joins, _, _ = self.query.setup_joins([f.name], opts, root_alias)
1257 alias = joins[-1]
1258 columns = self.get_default_columns(
1259 related_select_mask, start_alias=alias, opts=f.remote_field.model._meta
1260 )
1261 for col in columns:
1262 select_fields.append(len(select))
1263 select.append((col, None))
1264 klass_info["select_fields"] = select_fields
1265 next_klass_infos = self.get_related_selections(
1266 select,
1267 related_select_mask,
1268 f.remote_field.model._meta,
1269 alias,
1270 cur_depth + 1,
1271 next,
1272 restricted,
1273 )
1274 get_related_klass_infos(klass_info, next_klass_infos)
1275
1276 if restricted:
1277 related_fields = [
1278 (o, o.field, o.related_model)
1279 for o in opts.related_objects
1280 if o.field.unique and not o.many_to_many
1281 ]
1282 for related_object, related_field, model in related_fields:
1283 if not select_related_descend(
1284 related_object,
1285 restricted,
1286 requested,
1287 select_mask,
1288 ):
1289 continue
1290
1291 related_select_mask = select_mask.get(related_object) or {}
1292 related_field_name = related_field.related_query_name()
1293 fields_found.add(related_field_name)
1294
1295 join_info = self.query.setup_joins(
1296 [related_field_name], opts, root_alias
1297 )
1298 alias = join_info.joins[-1]
1299 from_parent = issubclass(model, opts.model) and model is not opts.model
1300 klass_info = {
1301 "model": model,
1302 "field": related_field,
1303 "reverse": True,
1304 "local_setter": related_object.set_cached_value,
1305 "remote_setter": related_field.set_cached_value,
1306 "from_parent": from_parent,
1307 }
1308 related_klass_infos.append(klass_info)
1309 select_fields = []
1310 columns = self.get_default_columns(
1311 related_select_mask,
1312 start_alias=alias,
1313 opts=model._meta,
1314 from_parent=opts.model,
1315 )
1316 for col in columns:
1317 select_fields.append(len(select))
1318 select.append((col, None))
1319 klass_info["select_fields"] = select_fields
1320 next = requested.get(related_field_name, {})
1321 next_klass_infos = self.get_related_selections(
1322 select,
1323 related_select_mask,
1324 model._meta,
1325 alias,
1326 cur_depth + 1,
1327 next,
1328 restricted,
1329 )
1330 get_related_klass_infos(klass_info, next_klass_infos)
1331
1332 def local_setter(final_field, obj, from_obj):
1333 # Set a reverse fk object when relation is non-empty.
1334 if from_obj:
1335 final_field.remote_field.set_cached_value(from_obj, obj)
1336
1337 def local_setter_noop(obj, from_obj):
1338 pass
1339
1340 def remote_setter(name, obj, from_obj):
1341 setattr(from_obj, name, obj)
1342
1343 for name in list(requested):
1344 # Filtered relations work only on the topmost level.
1345 if cur_depth > 1:
1346 break
1347 if name in self.query._filtered_relations:
1348 fields_found.add(name)
1349 final_field, _, join_opts, joins, _, _ = self.query.setup_joins(
1350 [name], opts, root_alias
1351 )
1352 model = join_opts.model
1353 alias = joins[-1]
1354 from_parent = (
1355 issubclass(model, opts.model) and model is not opts.model
1356 )
1357 klass_info = {
1358 "model": model,
1359 "field": final_field,
1360 "reverse": True,
1361 "local_setter": (
1362 partial(local_setter, final_field)
1363 if len(joins) <= 2
1364 else local_setter_noop
1365 ),
1366 "remote_setter": partial(remote_setter, name),
1367 "from_parent": from_parent,
1368 }
1369 related_klass_infos.append(klass_info)
1370 select_fields = []
1371 field_select_mask = select_mask.get((name, final_field)) or {}
1372 columns = self.get_default_columns(
1373 field_select_mask,
1374 start_alias=alias,
1375 opts=model._meta,
1376 from_parent=opts.model,
1377 )
1378 for col in columns:
1379 select_fields.append(len(select))
1380 select.append((col, None))
1381 klass_info["select_fields"] = select_fields
1382 next_requested = requested.get(name, {})
1383 next_klass_infos = self.get_related_selections(
1384 select,
1385 field_select_mask,
1386 opts=model._meta,
1387 root_alias=alias,
1388 cur_depth=cur_depth + 1,
1389 requested=next_requested,
1390 restricted=restricted,
1391 )
1392 get_related_klass_infos(klass_info, next_klass_infos)
1393 fields_not_found = set(requested).difference(fields_found)
1394 if fields_not_found:
1395 invalid_fields = ("'%s'" % s for s in fields_not_found)
1396 raise FieldError(
1397 "Invalid field name(s) given in select_related: %s. "
1398 "Choices are: %s"
1399 % (
1400 ", ".join(invalid_fields),
1401 ", ".join(_get_field_choices()) or "(none)",
1402 )
1403 )
1404 return related_klass_infos
1405
1406 def get_select_for_update_of_arguments(self):
1407 """
1408 Return a quoted list of arguments for the SELECT FOR UPDATE OF part of
1409 the query.
1410 """
1411
1412 def _get_parent_klass_info(klass_info):
1413 concrete_model = klass_info["model"]._meta.concrete_model
1414 for parent_model, parent_link in concrete_model._meta.parents.items():
1415 all_parents = parent_model._meta.all_parents
1416 yield {
1417 "model": parent_model,
1418 "field": parent_link,
1419 "reverse": False,
1420 "select_fields": [
1421 select_index
1422 for select_index in klass_info["select_fields"]
1423 # Selected columns from a model or its parents.
1424 if (
1425 self.select[select_index][0].target.model == parent_model
1426 or self.select[select_index][0].target.model in all_parents
1427 )
1428 ],
1429 }
1430
1431 def _get_first_selected_col_from_model(klass_info):
1432 """
1433 Find the first selected column from a model. If it doesn't exist,
1434 don't lock a model.
1435
1436 select_fields is filled recursively, so it also contains fields
1437 from the parent models.
1438 """
1439 concrete_model = klass_info["model"]._meta.concrete_model
1440 for select_index in klass_info["select_fields"]:
1441 if self.select[select_index][0].target.model == concrete_model:
1442 return self.select[select_index][0]
1443
1444 def _get_field_choices():
1445 """Yield all allowed field paths in breadth-first search order."""
1446 queue = collections.deque([(None, self.klass_info)])
1447 while queue:
1448 parent_path, klass_info = queue.popleft()
1449 if parent_path is None:
1450 path = []
1451 yield "self"
1452 else:
1453 field = klass_info["field"]
1454 if klass_info["reverse"]:
1455 field = field.remote_field
1456 path = parent_path + [field.name]
1457 yield LOOKUP_SEP.join(path)
1458 queue.extend(
1459 (path, klass_info)
1460 for klass_info in _get_parent_klass_info(klass_info)
1461 )
1462 queue.extend(
1463 (path, klass_info)
1464 for klass_info in klass_info.get("related_klass_infos", [])
1465 )
1466
1467 if not self.klass_info:
1468 return []
1469 result = []
1470 invalid_names = []
1471 for name in self.query.select_for_update_of:
1472 klass_info = self.klass_info
1473 if name == "self":
1474 col = _get_first_selected_col_from_model(klass_info)
1475 else:
1476 for part in name.split(LOOKUP_SEP):
1477 klass_infos = (
1478 *klass_info.get("related_klass_infos", []),
1479 *_get_parent_klass_info(klass_info),
1480 )
1481 for related_klass_info in klass_infos:
1482 field = related_klass_info["field"]
1483 if related_klass_info["reverse"]:
1484 field = field.remote_field
1485 if field.name == part:
1486 klass_info = related_klass_info
1487 break
1488 else:
1489 klass_info = None
1490 break
1491 if klass_info is None:
1492 invalid_names.append(name)
1493 continue
1494 col = _get_first_selected_col_from_model(klass_info)
1495 if col is not None:
1496 if self.connection.features.select_for_update_of_column:
1497 result.append(self.compile(col)[0])
1498 else:
1499 result.append(self.quote_name_unless_alias(col.alias))
1500 if invalid_names:
1501 raise FieldError(
1502 "Invalid field name(s) given in select_for_update(of=(...)): %s. "
1503 "Only relational fields followed in the query are allowed. "
1504 "Choices are: %s."
1505 % (
1506 ", ".join(invalid_names),
1507 ", ".join(_get_field_choices()),
1508 )
1509 )
1510 return result
1511
1512 def get_converters(self, expressions):
1513 i = 0
1514 converters = {}
1515
1516 for expression in expressions:
1517 if isinstance(expression, ColPairs):
1518 cols = expression.get_source_expressions()
1519 cols_converters = self.get_converters(cols)
1520 for j, (convs, col) in cols_converters.items():
1521 converters[i + j] = (convs, col)
1522 i += len(expression)
1523 elif expression:
1524 backend_converters = self.connection.ops.get_db_converters(expression)
1525 field_converters = expression.get_db_converters(self.connection)
1526 if backend_converters or field_converters:
1527 converters[i] = (backend_converters + field_converters, expression)
1528 i += 1
1529 else:
1530 i += 1
1531
1532 return converters
1533
1534 def apply_converters(self, rows, converters):
1535 connection = self.connection
1536 converters = list(converters.items())
1537 for row in map(list, rows):
1538 for pos, (convs, expression) in converters:
1539 value = row[pos]
1540 for converter in convs:
1541 value = converter(value, expression, connection)
1542 row[pos] = value
1543 yield row
1544
1545 def has_composite_fields(self, expressions):
1546 # Check for composite fields before calling the relatively costly
1547 # composite_fields_to_tuples.
1548 return any(isinstance(expression, ColPairs) for expression in expressions)
1549
1550 def composite_fields_to_tuples(self, rows, expressions):
1551 col_pair_slices = [
1552 slice(i, i + len(expression))
1553 for i, expression in enumerate(expressions)
1554 if isinstance(expression, ColPairs)
1555 ]
1556
1557 for row in map(list, rows):
1558 for pos in col_pair_slices:
1559 row[pos] = (tuple(row[pos]),)
1560
1561 yield row
1562
1563 def results_iter(
1564 self,
1565 results=None,
1566 tuple_expected=False,
1567 chunked_fetch=False,
1568 chunk_size=GET_ITERATOR_CHUNK_SIZE,
1569 ):
1570 """Return an iterator over the results from executing this query."""
1571 if results is None:
1572 results = self.execute_sql(
1573 MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
1574 )
1575 fields = [s[0] for s in self.select[0 : self.col_count]]
1576 converters = self.get_converters(fields)
1577 rows = chain.from_iterable(results)
1578 if converters:
1579 rows = self.apply_converters(rows, converters)
1580 if self.has_composite_fields(fields):
1581 rows = self.composite_fields_to_tuples(rows, fields)
1582 if tuple_expected:
1583 rows = map(tuple, rows)
1584 return rows
1585
1586 def has_results(self):
1587 """
1588 Backends (e.g. NoSQL) can override this in order to use optimized
1589 versions of "query has any results."
1590 """
1591 return bool(self.execute_sql(SINGLE))
1592
1593 def execute_sql(
1594 self, result_type=MULTI, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE
1595 ):
1596 """
1597 Run the query against the database and return the result(s). The
1598 return value depends on the value of result_type.
1599
1600 When result_type is:
1601 - MULTI: Retrieves all rows using fetchmany(). Wraps in an iterator for
1602 chunked reads when supported.
1603 - SINGLE: Retrieves a single row using fetchone().
1604 - ROW_COUNT: Retrieves the number of rows in the result.
1605 - CURSOR: Runs the query, and returns the cursor object. It is the
1606 caller's responsibility to close the cursor.
1607 """
1608 result_type = result_type or NO_RESULTS
1609 try:
1610 sql, params = self.as_sql()
1611 if not sql:
1612 raise EmptyResultSet
1613 except EmptyResultSet:
1614 if result_type == MULTI:
1615 return iter([])
1616 else:
1617 return
1618 if chunked_fetch:
1619 cursor = self.connection.chunked_cursor()
1620 else:
1621 cursor = self.connection.cursor()
1622 try:
1623 cursor.execute(sql, params)
1624 except Exception:
1625 # Might fail for server-side cursors (e.g. connection closed)
1626 cursor.close()
1627 raise
1628
1629 if result_type == ROW_COUNT:
1630 try:
1631 return cursor.rowcount
1632 finally:
1633 cursor.close()
1634 if result_type == CURSOR:
1635 # Give the caller the cursor to process and close.
1636 return cursor
1637 if result_type == SINGLE:
1638 try:
1639 val = cursor.fetchone()
1640 if val:
1641 return val[0 : self.col_count]
1642 return val
1643 finally:
1644 # done with the cursor
1645 cursor.close()
1646 if result_type == NO_RESULTS:
1647 cursor.close()
1648 return
1649
1650 result = cursor_iter(
1651 cursor,
1652 self.connection.features.empty_fetchmany_value,
1653 self.col_count if self.has_extra_select else None,
1654 chunk_size,
1655 )
1656 if not chunked_fetch or not self.connection.features.can_use_chunked_reads:
1657 # If we are using non-chunked reads, we return the same data
1658 # structure as normally, but ensure it is all read into memory
1659 # before going any further. Use chunked_fetch if requested,
1660 # unless the database doesn't support it.
1661 return list(result)
1662 return result
1663
1664 def explain_query(self):
1665 result = list(self.execute_sql())
1666 # Some backends return 1 item tuples with strings, and others return
1667 # tuples with integers and strings. Flatten them out into strings.
1668 format_ = self.query.explain_info.format
1669 output_formatter = json.dumps if format_ and format_.lower() == "json" else str
1670 for row in result:
1671 for value in row:
1672 if not isinstance(value, str):
1673 yield " ".join([output_formatter(c) for c in value])
1674 else:
1675 yield value
1676
1677
1678 class SQLInsertCompiler(SQLCompiler):
1679 returning_fields = None
1680 returning_params = ()
1681
1682 def field_as_sql(self, field, get_placeholder, val):
1683 """
1684 Take a field and a value intended to be saved on that field, and
1685 return placeholder SQL and accompanying params. Check for raw values,
1686 expressions, and fields with get_placeholder() defined in that order.
1687
1688 When field is None, consider the value raw and use it as the
1689 placeholder, with no corresponding parameters returned.
1690 """
1691 if field is None:
1692 # A field value of None means the value is raw.
1693 sql, params = val, []
1694 elif hasattr(val, "as_sql"):
1695 # This is an expression, let's compile it.
1696 sql, params = self.compile(val)
1697 elif get_placeholder is not None:
1698 # Some fields (e.g. geo fields) need special munging before
1699 # they can be inserted.
1700 sql, params = get_placeholder(val, self, self.connection), [val]
1701 else:
1702 # Return the common case for the placeholder
1703 sql, params = "%s", [val]
1704
1705 # The following hook is only used by Oracle Spatial, which sometimes
1706 # needs to yield 'NULL' and [] as its placeholder and params instead
1707 # of '%s' and [None]. The 'NULL' placeholder is produced earlier by
1708 # OracleOperations.get_geom_placeholder(). The following line removes
1709 # the corresponding None parameter. See ticket #10888.
1710 params = self.connection.ops.modify_insert_params(sql, params)
1711
1712 return sql, params
1713
1714 def prepare_value(self, field, value):
1715 """
1716 Prepare a value to be used in a query by resolving it if it is an
1717 expression and otherwise calling the field's get_db_prep_save().
1718 """
1719 if hasattr(value, "resolve_expression"):
1720 value = value.resolve_expression(
1721 self.query, allow_joins=False, for_save=True
1722 )
1723 # Don't allow values containing Col expressions. They refer to
1724 # existing columns on a row, but in the case of insert the row
1725 # doesn't exist yet.
1726 if value.contains_column_references:
1727 raise ValueError(
1728 'Failed to insert expression "%s" on %s. F() expressions '
1729 "can only be used to update, not to insert." % (value, field)
1730 )
1731 if value.contains_aggregate:
1732 raise FieldError(
1733 "Aggregate functions are not allowed in this query "
1734 "(%s=%r)." % (field.name, value)
1735 )
1736 if value.contains_over_clause:
1737 raise FieldError(
1738 "Window expressions are not allowed in this query (%s=%r)."
1739 % (field.name, value)
1740 )
1741 return field.get_db_prep_save(value, connection=self.connection)
1742
1743 def pre_save_val(self, field, obj):
1744 """
1745 Get the given field's value off the given obj. pre_save() is used for
1746 things like auto_now on DateTimeField. Skip it if this is a raw query.
1747 """
1748 if self.query.raw:
1749 return getattr(obj, field.attname)
1750 return field.pre_save(obj, add=True)
1751
1752 def assemble_as_sql(self, fields, value_rows):
1753 """
1754 Take a sequence of N fields and a sequence of M rows of values, and
1755 generate placeholder SQL and parameters for each field and value.
1756 Return a pair containing:
1757 * a sequence of M rows of N SQL placeholder strings, and
1758 * a sequence of M rows of corresponding parameter values.
1759
1760 Each placeholder string may contain any number of '%s' interpolation
1761 strings, and each parameter row will contain exactly as many params
1762 as the total number of '%s's in the corresponding placeholder row.
1763 """
1764 if not value_rows:
1765 return [], []
1766
1767 # list of (sql, [params]) tuples for each object to be saved
1768 # Shape: [n_objs][n_fields][2]
1769 get_placeholders = [getattr(field, "get_placeholder", None) for field in fields]
1770 rows_of_fields_as_sql = (
1771 (
1772 self.field_as_sql(field, get_placeholder, value)
1773 for field, get_placeholder, value in zip(fields, get_placeholders, row)
1774 )
1775 for row in value_rows
1776 )
1777
1778 # tuple like ([sqls], [[params]s]) for each object to be saved
1779 # Shape: [n_objs][2][n_fields]
1780 sql_and_param_pair_rows = (zip(*row) for row in rows_of_fields_as_sql)
1781
1782 # Extract separate lists for placeholders and params.
1783 # Each of these has shape [n_objs][n_fields]
1784 placeholder_rows, param_rows = zip(*sql_and_param_pair_rows)
1785
1786 # Params for each field are still lists, and need to be flattened.
1787 param_rows = [[p for ps in row for p in ps] for row in param_rows]
1788
1789 return placeholder_rows, param_rows
1790
1791 def as_sql(self):
1792 # We don't need quote_name_unless_alias() here, since these are all
1793 # going to be column names (so we can avoid the extra overhead).
1794 qn = self.connection.ops.quote_name
1795 opts = self.query.get_meta()
1796 insert_statement = self.connection.ops.insert_statement(
1797 on_conflict=self.query.on_conflict,
1798 )
1799 result = ["%s %s" % (insert_statement, qn(opts.db_table))]
1800 fields = self.query.fields or [opts.pk]
1801 result.append("(%s)" % ", ".join(qn(f.column) for f in fields))
1802
1803 if self.query.fields:
1804 value_rows = [
1805 [
1806 self.prepare_value(field, self.pre_save_val(field, obj))
1807 for field in fields
1808 ]
1809 for obj in self.query.objs
1810 ]
1811 else:
1812 # An empty object.
1813 value_rows = [
1814 [self.connection.ops.pk_default_value()] for _ in self.query.objs
1815 ]
1816 fields = [None]
1817
1818 # Currently the backends just accept values when generating bulk
1819 # queries and generate their own placeholders. Doing that isn't
1820 # necessary and it should be possible to use placeholders and
1821 # expressions in bulk inserts too.
1822 can_bulk = (
1823 not self.returning_fields and self.connection.features.has_bulk_insert
1824 )
1825
1826 placeholder_rows, param_rows = self.assemble_as_sql(fields, value_rows)
1827
1828 on_conflict_suffix_sql = self.connection.ops.on_conflict_suffix_sql(
1829 fields,
1830 self.query.on_conflict,
1831 (f.column for f in self.query.update_fields),
1832 (f.column for f in self.query.unique_fields),
1833 )
1834 if (
1835 self.returning_fields
1836 and self.connection.features.can_return_columns_from_insert
1837 ):
1838 if self.connection.features.can_return_rows_from_bulk_insert:
1839 result.append(
1840 self.connection.ops.bulk_insert_sql(fields, placeholder_rows)
1841 )
1842 params = param_rows
1843 else:
1844 result.append("VALUES (%s)" % ", ".join(placeholder_rows[0]))
1845 params = [param_rows[0]]
1846 if on_conflict_suffix_sql:
1847 result.append(on_conflict_suffix_sql)
1848 # Skip empty r_sql to allow subclasses to customize behavior for
1849 # 3rd party backends. Refs #19096.
1850 r_sql, self.returning_params = self.connection.ops.return_insert_columns(
1851 self.returning_fields
1852 )
1853 if r_sql:
1854 result.append(r_sql)
1855 params += [self.returning_params]
1856 return [(" ".join(result), tuple(chain.from_iterable(params)))]
1857
1858 if can_bulk:
1859 result.append(self.connection.ops.bulk_insert_sql(fields, placeholder_rows))
1860 if on_conflict_suffix_sql:
1861 result.append(on_conflict_suffix_sql)
1862 return [(" ".join(result), tuple(p for ps in param_rows for p in ps))]
1863 else:
1864 if on_conflict_suffix_sql:
1865 result.append(on_conflict_suffix_sql)
1866 return [
1867 (" ".join(result + ["VALUES (%s)" % ", ".join(p)]), vals)
1868 for p, vals in zip(placeholder_rows, param_rows)
1869 ]
1870
1871 def execute_sql(self, returning_fields=None):
1872 assert not (
1873 returning_fields
1874 and len(self.query.objs) != 1
1875 and not self.connection.features.can_return_rows_from_bulk_insert
1876 )
1877 opts = self.query.get_meta()
1878 self.returning_fields = returning_fields
1879 cols = []
1880 with self.connection.cursor() as cursor:
1881 for sql, params in self.as_sql():
1882 cursor.execute(sql, params)
1883 if not self.returning_fields:
1884 return []
1885 if (
1886 self.connection.features.can_return_rows_from_bulk_insert
1887 and len(self.query.objs) > 1
1888 ):
1889 rows = self.connection.ops.fetch_returned_insert_rows(cursor)
1890 cols = [field.get_col(opts.db_table) for field in self.returning_fields]
1891 elif self.connection.features.can_return_columns_from_insert:
1892 assert len(self.query.objs) == 1
1893 rows = [
1894 self.connection.ops.fetch_returned_insert_columns(
1895 cursor,
1896 self.returning_params,
1897 )
1898 ]
1899 cols = [field.get_col(opts.db_table) for field in self.returning_fields]
1900 elif returning_fields and isinstance(
1901 returning_field := returning_fields[0], AutoField
1902 ):
1903 cols = [returning_field.get_col(opts.db_table)]
1904 rows = [
1905 (
1906 self.connection.ops.last_insert_id(
1907 cursor,
1908 opts.db_table,
1909 returning_field.column,
1910 ),
1911 )
1912 ]
1913 else:
1914 # Backend doesn't support returning fields and no auto-field
1915 # that can be retrieved from `last_insert_id` was specified.
1916 return []
1917 converters = self.get_converters(cols)
1918 if converters:
1919 rows = self.apply_converters(rows, converters)
1920 return list(rows)
1921
1922
1923 class SQLDeleteCompiler(SQLCompiler):
1924 @cached_property
1925 def single_alias(self):
1926 # Ensure base table is in aliases.
1927 self.query.get_initial_alias()
1928 return sum(self.query.alias_refcount[t] > 0 for t in self.query.alias_map) == 1
1929
1930 @classmethod
1931 def _expr_refs_base_model(cls, expr, base_model):
1932 if isinstance(expr, Query):
1933 return expr.model == base_model
1934 if not hasattr(expr, "get_source_expressions"):
1935 return False
1936 return any(
1937 cls._expr_refs_base_model(source_expr, base_model)
1938 for source_expr in expr.get_source_expressions()
1939 )
1940
1941 @cached_property
1942 def contains_self_reference_subquery(self):
1943 return any(
1944 self._expr_refs_base_model(expr, self.query.model)
1945 for expr in chain(
1946 self.query.annotations.values(), self.query.where.children
1947 )
1948 )
1949
1950 def _as_sql(self, query):
1951 delete = "DELETE FROM %s" % self.quote_name_unless_alias(query.base_table)
1952 try:
1953 where, params = self.compile(query.where)
1954 except FullResultSet:
1955 return delete, ()
1956 return f"{delete} WHERE {where}", tuple(params)
1957
1958 def as_sql(self):
1959 """
1960 Create the SQL for this query. Return the SQL string and list of
1961 parameters.
1962 """
1963 if self.single_alias and (
1964 self.connection.features.delete_can_self_reference_subquery
1965 or not self.contains_self_reference_subquery
1966 ):
1967 return self._as_sql(self.query)
1968 innerq = self.query.clone()
1969 innerq.__class__ = Query
1970 innerq.clear_select_clause()
1971 pk = self.query.model._meta.pk
1972 innerq.select = [pk.get_col(self.query.get_initial_alias())]
1973 outerq = Query(self.query.model)
1974 if not self.connection.features.update_can_self_select:
1975 # Force the materialization of the inner query to allow reference
1976 # to the target table on MySQL.
1977 sql, params = innerq.get_compiler(connection=self.connection).as_sql()
1978 innerq = RawSQL("SELECT * FROM (%s) subquery" % sql, params)
1979 outerq.add_filter("pk__in", innerq)
1980 return self._as_sql(outerq)
1981
1982
1983 class SQLUpdateCompiler(SQLCompiler):
1984 def as_sql(self):
1985 """
1986 Create the SQL for this query. Return the SQL string and list of
1987 parameters.
1988 """
1989 self.pre_sql_setup()
1990 if not self.query.values:
1991 return "", ()
1992 qn = self.quote_name_unless_alias
1993 values, update_params = [], []
1994 for field, model, val in self.query.values:
1995 if hasattr(val, "resolve_expression"):
1996 val = val.resolve_expression(
1997 self.query, allow_joins=False, for_save=True
1998 )
1999 if val.contains_aggregate:
2000 raise FieldError(
2001 "Aggregate functions are not allowed in this query "
2002 "(%s=%r)." % (field.name, val)
2003 )
2004 if val.contains_over_clause:
2005 raise FieldError(
2006 "Window expressions are not allowed in this query "
2007 "(%s=%r)." % (field.name, val)
2008 )
2009 if isinstance(val, ColPairs):
2010 raise FieldError(
2011 "Composite primary keys expressions are not allowed "
2012 "in this query (%s=F('pk'))." % field.name
2013 )
2014 elif hasattr(val, "prepare_database_save"):
2015 if field.remote_field:
2016 val = val.prepare_database_save(field)
2017 else:
2018 raise TypeError(
2019 "Tried to update field %s with a model instance, %r. "
2020 "Use a value compatible with %s."
2021 % (field, val, field.__class__.__name__)
2022 )
2023 val = field.get_db_prep_save(val, connection=self.connection)
2024
2025 # Getting the placeholder for the field.
2026 if hasattr(field, "get_placeholder"):
2027 placeholder = field.get_placeholder(val, self, self.connection)
2028 else:
2029 placeholder = "%s"
2030 name = field.column
2031 if hasattr(val, "as_sql"):
2032 sql, params = self.compile(val)
2033 values.append("%s = %s" % (qn(name), placeholder % sql))
2034 update_params.extend(params)
2035 elif val is not None:
2036 values.append("%s = %s" % (qn(name), placeholder))
2037 update_params.append(val)
2038 else:
2039 values.append("%s = NULL" % qn(name))
2040 table = self.query.base_table
2041 result = [
2042 "UPDATE %s SET" % qn(table),
2043 ", ".join(values),
2044 ]
2045 try:
2046 where, params = self.compile(self.query.where)
2047 except FullResultSet:
2048 params = []
2049 else:
2050 result.append("WHERE %s" % where)
2051 return " ".join(result), tuple(update_params + params)
2052
2053 def execute_sql(self, result_type):
2054 """
2055 Execute the specified update. Return the number of rows affected by
2056 the primary update query. The "primary update query" is the first
2057 non-empty query that is executed. Row counts for any subsequent,
2058 related queries are not available.
2059 """
2060 row_count = super().execute_sql(result_type)
2061 is_empty = row_count is None
2062 row_count = row_count or 0
2063
2064 for query in self.query.get_related_updates():
2065 # If the result_type is NO_RESULTS then the aux_row_count is None.
2066 aux_row_count = query.get_compiler(self.using).execute_sql(result_type)
2067 if is_empty and aux_row_count:
2068 # Returns the row count for any related updates as the number of
2069 # rows updated.
2070 row_count = aux_row_count
2071 is_empty = False
2072 return row_count
2073
2074 def pre_sql_setup(self):
2075 """
2076 If the update depends on results from other tables, munge the "where"
2077 conditions to match the format required for (portable) SQL updates.
2078
2079 If multiple updates are required, pull out the id values to update at
2080 this point so that they don't change as a result of the progressive
2081 updates.
2082 """
2083 refcounts_before = self.query.alias_refcount.copy()
2084 # Ensure base table is in the query
2085 self.query.get_initial_alias()
2086 count = self.query.count_active_tables()
2087 if not self.query.related_updates and count == 1:
2088 return
2089 query = self.query.chain(klass=Query)
2090 query.select_related = False
2091 query.clear_ordering(force=True)
2092 query.extra = {}
2093 query.select = []
2094 meta = query.get_meta()
2095 fields = [meta.pk.name]
2096 related_ids_index = []
2097 for related in self.query.related_updates:
2098 if all(
2099 path.join_field.primary_key for path in meta.get_path_to_parent(related)
2100 ):
2101 # If a primary key chain exists to the targeted related update,
2102 # then the meta.pk value can be used for it.
2103 related_ids_index.append((related, 0))
2104 else:
2105 # This branch will only be reached when updating a field of an
2106 # ancestor that is not part of the primary key chain of a MTI
2107 # tree.
2108 related_ids_index.append((related, len(fields)))
2109 fields.append(related._meta.pk.name)
2110 query.add_fields(fields)
2111 super().pre_sql_setup()
2112
2113 is_composite_pk = meta.is_composite_pk
2114 must_pre_select = (
2115 count > 1 and not self.connection.features.update_can_self_select
2116 )
2117
2118 # Now we adjust the current query: reset the where clause and get rid
2119 # of all the tables we don't need (since they're in the sub-select).
2120 self.query.clear_where()
2121 if self.query.related_updates or must_pre_select:
2122 # Either we're using the idents in multiple update queries (so
2123 # don't want them to change), or the db backend doesn't support
2124 # selecting from the updating table (e.g. MySQL).
2125 idents = []
2126 related_ids = collections.defaultdict(list)
2127 for rows in query.get_compiler(self.using).execute_sql(MULTI):
2128 pks = [row if is_composite_pk else row[0] for row in rows]
2129 idents.extend(pks)
2130 for parent, index in related_ids_index:
2131 related_ids[parent].extend(r[index] for r in rows)
2132 self.query.add_filter("pk__in", idents)
2133 self.query.related_ids = related_ids
2134 else:
2135 # The fast path. Filters and updates in one query.
2136 self.query.add_filter("pk__in", query)
2137 self.query.reset_refcounts(refcounts_before)
2138
2139
2140 class SQLAggregateCompiler(SQLCompiler):
2141 def as_sql(self):
2142 """
2143 Create the SQL for this query. Return the SQL string and list of
2144 parameters.
2145 """
2146 sql, params = [], []
2147 for annotation in self.query.annotation_select.values():
2148 ann_sql, ann_params = self.compile(annotation)
2149 ann_sql, ann_params = annotation.select_format(self, ann_sql, ann_params)
2150 sql.append(ann_sql)
2151 params.extend(ann_params)
2152 self.col_count = len(self.query.annotation_select)
2153 sql = ", ".join(sql)
2154 params = tuple(params)
2155
2156 inner_query_sql, inner_query_params = self.query.inner_query.get_compiler(
2157 self.using,
2158 elide_empty=self.elide_empty,
2159 ).as_sql(with_col_aliases=True)
2160 sql = "SELECT %s FROM (%s) subquery" % (sql, inner_query_sql)
2161 params += inner_query_params
2162 return sql, params
2163
2164
2165 def cursor_iter(cursor, sentinel, col_count, itersize):
2166 """
2167 Yield blocks of rows from a cursor and ensure the cursor is closed when
2168 done.
2169 """
2170 try:
2171 for rows in iter((lambda: cursor.fetchmany(itersize)), sentinel):
2172 yield rows if col_count is None else [r[:col_count] for r in rows]
2173 finally:
2174 cursor.close()