| 1 |
""" |
| 2 |
Create SQL statements for QuerySets. |
| 3 |
|
| 4 |
The code in here encapsulates all of the SQL construction so that QuerySets |
| 5 |
themselves do not have to (and could be backed by things other than SQL |
| 6 |
databases). The abstraction barrier only works one way: this module has to know |
| 7 |
all about the internals of models in order to get the information it needs. |
| 8 |
""" |
| 9 |
|
| 10 |
import copy |
| 11 |
import difflib |
| 12 |
import functools |
| 13 |
import sys |
| 14 |
from collections import Counter, namedtuple |
| 15 |
from collections.abc import Iterator, Mapping |
| 16 |
from itertools import chain, count, product |
| 17 |
from string import ascii_uppercase |
| 18 |
|
| 19 |
from django.core.exceptions import FieldDoesNotExist, FieldError |
| 20 |
from django.db import DEFAULT_DB_ALIAS, NotSupportedError, connections |
| 21 |
from django.db.models.aggregates import Count |
| 22 |
from django.db.models.constants import LOOKUP_SEP |
| 23 |
from django.db.models.expressions import ( |
| 24 |
BaseExpression, |
| 25 |
Col, |
| 26 |
ColPairs, |
| 27 |
Exists, |
| 28 |
F, |
| 29 |
OuterRef, |
| 30 |
RawSQL, |
| 31 |
Ref, |
| 32 |
ResolvedOuterRef, |
| 33 |
Value, |
| 34 |
) |
| 35 |
from django.db.models.fields import Field |
| 36 |
from django.db.models.lookups import Lookup |
| 37 |
from django.db.models.query_utils import ( |
| 38 |
Q, |
| 39 |
check_rel_lookup_compatibility, |
| 40 |
refs_expression, |
| 41 |
) |
| 42 |
from django.db.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE |
| 43 |
from django.db.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin |
| 44 |
from django.db.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode |
| 45 |
from django.utils.functional import cached_property |
| 46 |
from django.utils.regex_helper import _lazy_re_compile |
| 47 |
from django.utils.tree import Node |
| 48 |
|
| 49 |
__all__ = ["Query", "RawQuery"] |
| 50 |
|
| 51 |
# Quotation marks ('"`[]), whitespace characters, control characters, |
| 52 |
# semicolons, hashes, or inline SQL comments are forbidden in column aliases. |
| 53 |
FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile( |
| 54 |
r"['`\"\]\[;\s\x00-\x1F\x7F-\x9F]|#|--|/\*|\*/" |
| 55 |
) |
| 56 |
|
| 57 |
# Inspired from |
| 58 |
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS |
| 59 |
EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w-]+") |
| 60 |
|
| 61 |
|
| 62 |
def get_field_names_from_opts(opts): |
| 63 |
if opts is None: |
| 64 |
return set() |
| 65 |
return set( |
| 66 |
chain.from_iterable( |
| 67 |
(f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields() |
| 68 |
) |
| 69 |
) |
| 70 |
|
| 71 |
|
| 72 |
def get_paths_from_expression(expr): |
| 73 |
if isinstance(expr, F): |
| 74 |
yield expr.name |
| 75 |
elif hasattr(expr, "flatten"): |
| 76 |
for child in expr.flatten(): |
| 77 |
if isinstance(child, F): |
| 78 |
yield child.name |
| 79 |
elif isinstance(child, Q): |
| 80 |
yield from get_children_from_q(child) |
| 81 |
|
| 82 |
|
| 83 |
def get_children_from_q(q): |
| 84 |
for child in q.children: |
| 85 |
if isinstance(child, Node): |
| 86 |
yield from get_children_from_q(child) |
| 87 |
elif isinstance(child, tuple): |
| 88 |
lhs, rhs = child |
| 89 |
yield lhs |
| 90 |
if hasattr(rhs, "resolve_expression"): |
| 91 |
yield from get_paths_from_expression(rhs) |
| 92 |
elif hasattr(child, "resolve_expression"): |
| 93 |
yield from get_paths_from_expression(child) |
| 94 |
|
| 95 |
|
| 96 |
def get_child_with_renamed_prefix(prefix, replacement, child): |
| 97 |
from django.db.models.query import QuerySet |
| 98 |
|
| 99 |
if isinstance(child, Node): |
| 100 |
return rename_prefix_from_q(prefix, replacement, child) |
| 101 |
if isinstance(child, tuple): |
| 102 |
lhs, rhs = child |
| 103 |
if lhs.startswith(prefix + LOOKUP_SEP): |
| 104 |
lhs = lhs.replace(prefix, replacement, 1) |
| 105 |
if not isinstance(rhs, F) and hasattr(rhs, "resolve_expression"): |
| 106 |
rhs = get_child_with_renamed_prefix(prefix, replacement, rhs) |
| 107 |
return lhs, rhs |
| 108 |
|
| 109 |
if isinstance(child, F): |
| 110 |
child = child.copy() |
| 111 |
if child.name.startswith(prefix + LOOKUP_SEP): |
| 112 |
child.name = child.name.replace(prefix, replacement, 1) |
| 113 |
elif isinstance(child, QuerySet): |
| 114 |
# QuerySet may contain OuterRef() references which cannot work properly |
| 115 |
# without repointing to the filtered annotation and will spawn a |
| 116 |
# different JOIN. Always raise ValueError instead of providing partial |
| 117 |
# support in other cases. |
| 118 |
raise ValueError( |
| 119 |
"Passing a QuerySet within a FilteredRelation is not supported." |
| 120 |
) |
| 121 |
elif hasattr(child, "resolve_expression"): |
| 122 |
child = child.copy() |
| 123 |
child.set_source_expressions( |
| 124 |
[ |
| 125 |
get_child_with_renamed_prefix(prefix, replacement, grand_child) |
| 126 |
for grand_child in child.get_source_expressions() |
| 127 |
] |
| 128 |
) |
| 129 |
return child |
| 130 |
|
| 131 |
|
| 132 |
def rename_prefix_from_q(prefix, replacement, q): |
| 133 |
return Q.create( |
| 134 |
[get_child_with_renamed_prefix(prefix, replacement, c) for c in q.children], |
| 135 |
q.connector, |
| 136 |
q.negated, |
| 137 |
) |
| 138 |
|
| 139 |
|
| 140 |
JoinInfo = namedtuple( |
| 141 |
"JoinInfo", |
| 142 |
("final_field", "targets", "opts", "joins", "path", "transform_function"), |
| 143 |
) |
| 144 |
|
| 145 |
|
| 146 |
class RawQuery: |
| 147 |
"""A single raw SQL query.""" |
| 148 |
|
| 149 |
def __init__(self, sql, using, params=()): |
| 150 |
self.params = params |
| 151 |
self.sql = sql |
| 152 |
self.using = using |
| 153 |
self.cursor = None |
| 154 |
|
| 155 |
# Mirror some properties of a normal query so that |
| 156 |
# the compiler can be used to process results. |
| 157 |
self.low_mark, self.high_mark = 0, None # Used for offset/limit |
| 158 |
self.extra_select = {} |
| 159 |
self.annotation_select = {} |
| 160 |
|
| 161 |
def chain(self, using): |
| 162 |
return self.clone(using) |
| 163 |
|
| 164 |
def clone(self, using): |
| 165 |
return RawQuery(self.sql, using, params=self.params) |
| 166 |
|
| 167 |
def get_columns(self): |
| 168 |
if self.cursor is None: |
| 169 |
self._execute_query() |
| 170 |
converter = connections[self.using].introspection.identifier_converter |
| 171 |
return [converter(column_meta[0]) for column_meta in self.cursor.description] |
| 172 |
|
| 173 |
def __iter__(self): |
| 174 |
# Always execute a new query for a new iterator. |
| 175 |
# This could be optimized with a cache at the expense of RAM. |
| 176 |
self._execute_query() |
| 177 |
if not connections[self.using].features.can_use_chunked_reads: |
| 178 |
# If the database can't use chunked reads we need to make sure we |
| 179 |
# evaluate the entire query up front. |
| 180 |
result = list(self.cursor) |
| 181 |
else: |
| 182 |
result = self.cursor |
| 183 |
return iter(result) |
| 184 |
|
| 185 |
def __repr__(self): |
| 186 |
return "<%s: %s>" % (self.__class__.__name__, self) |
| 187 |
|
| 188 |
@property |
| 189 |
def params_type(self): |
| 190 |
if self.params is None: |
| 191 |
return None |
| 192 |
return dict if isinstance(self.params, Mapping) else tuple |
| 193 |
|
| 194 |
def __str__(self): |
| 195 |
if self.params_type is None: |
| 196 |
return self.sql |
| 197 |
return self.sql % self.params_type(self.params) |
| 198 |
|
| 199 |
def _execute_query(self): |
| 200 |
connection = connections[self.using] |
| 201 |
|
| 202 |
# Adapt parameters to the database, as much as possible considering |
| 203 |
# that the target type isn't known. See #17755. |
| 204 |
params_type = self.params_type |
| 205 |
adapter = connection.ops.adapt_unknown_value |
| 206 |
if params_type is tuple: |
| 207 |
params = tuple(adapter(val) for val in self.params) |
| 208 |
elif params_type is dict: |
| 209 |
params = {key: adapter(val) for key, val in self.params.items()} |
| 210 |
elif params_type is None: |
| 211 |
params = None |
| 212 |
else: |
| 213 |
raise RuntimeError("Unexpected params type: %s" % params_type) |
| 214 |
|
| 215 |
self.cursor = connection.cursor() |
| 216 |
self.cursor.execute(self.sql, params) |
| 217 |
|
| 218 |
|
| 219 |
ExplainInfo = namedtuple("ExplainInfo", ("format", "options")) |
| 220 |
|
| 221 |
|
| 222 |
class Query(BaseExpression): |
| 223 |
"""A single SQL query.""" |
| 224 |
|
| 225 |
alias_prefix = "T" |
| 226 |
empty_result_set_value = None |
| 227 |
subq_aliases = frozenset([alias_prefix]) |
| 228 |
|
| 229 |
compiler = "SQLCompiler" |
| 230 |
|
| 231 |
base_table_class = BaseTable |
| 232 |
join_class = Join |
| 233 |
|
| 234 |
default_cols = True |
| 235 |
default_ordering = True |
| 236 |
standard_ordering = True |
| 237 |
|
| 238 |
filter_is_sticky = False |
| 239 |
subquery = False |
| 240 |
|
| 241 |
# SQL-related attributes. |
| 242 |
# Select and related select clauses are expressions to use in the SELECT |
| 243 |
# clause of the query. The select is used for cases where we want to set up |
| 244 |
# the select clause to contain other than default fields (values(), |
| 245 |
# subqueries...). Note that annotations go to annotations dictionary. |
| 246 |
select = () |
| 247 |
# The group_by attribute can have one of the following forms: |
| 248 |
# - None: no group by at all in the query |
| 249 |
# - A tuple of expressions: group by (at least) those expressions. |
| 250 |
# String refs are also allowed for now. |
| 251 |
# - True: group by all select fields of the model |
| 252 |
# See compiler.get_group_by() for details. |
| 253 |
group_by = None |
| 254 |
order_by = () |
| 255 |
low_mark = 0 # Used for offset/limit. |
| 256 |
high_mark = None # Used for offset/limit. |
| 257 |
distinct = False |
| 258 |
distinct_fields = () |
| 259 |
select_for_update = False |
| 260 |
select_for_update_nowait = False |
| 261 |
select_for_update_skip_locked = False |
| 262 |
select_for_update_of = () |
| 263 |
select_for_no_key_update = False |
| 264 |
select_related = False |
| 265 |
# Arbitrary limit for select_related to prevents infinite recursion. |
| 266 |
max_depth = 5 |
| 267 |
# Holds the selects defined by a call to values() or values_list() |
| 268 |
# excluding annotation_select and extra_select. |
| 269 |
values_select = () |
| 270 |
selected = None |
| 271 |
|
| 272 |
# SQL annotation-related attributes. |
| 273 |
annotation_select_mask = None |
| 274 |
_annotation_select_cache = None |
| 275 |
|
| 276 |
# Set combination attributes. |
| 277 |
combinator = None |
| 278 |
combinator_all = False |
| 279 |
combined_queries = () |
| 280 |
|
| 281 |
# These are for extensions. The contents are more or less appended verbatim |
| 282 |
# to the appropriate clause. |
| 283 |
extra_select_mask = None |
| 284 |
_extra_select_cache = None |
| 285 |
|
| 286 |
extra_tables = () |
| 287 |
extra_order_by = () |
| 288 |
|
| 289 |
# A tuple that is a set of model field names and either True, if these are |
| 290 |
# the fields to defer, or False if these are the only fields to load. |
| 291 |
deferred_loading = (frozenset(), True) |
| 292 |
|
| 293 |
explain_info = None |
| 294 |
|
| 295 |
def __init__(self, model, alias_cols=True): |
| 296 |
self.model = model |
| 297 |
self.alias_refcount = {} |
| 298 |
# alias_map is the most important data structure regarding joins. |
| 299 |
# It's used for recording which joins exist in the query and what |
| 300 |
# types they are. The key is the alias of the joined table (possibly |
| 301 |
# the table name) and the value is a Join-like object (see |
| 302 |
# sql.datastructures.Join for more information). |
| 303 |
self.alias_map = {} |
| 304 |
# Whether to provide alias to columns during reference resolving. |
| 305 |
self.alias_cols = alias_cols |
| 306 |
# Sometimes the query contains references to aliases in outer queries (as |
| 307 |
# a result of split_exclude). Correct alias quoting needs to know these |
| 308 |
# aliases too. |
| 309 |
# Map external tables to whether they are aliased. |
| 310 |
self.external_aliases = {} |
| 311 |
self.table_map = {} # Maps table names to list of aliases. |
| 312 |
self.used_aliases = set() |
| 313 |
|
| 314 |
self.where = WhereNode() |
| 315 |
# Maps alias -> Annotation Expression. |
| 316 |
self.annotations = {} |
| 317 |
# These are for extensions. The contents are more or less appended |
| 318 |
# verbatim to the appropriate clause. |
| 319 |
self.extra = {} # Maps col_alias -> (col_sql, params). |
| 320 |
|
| 321 |
self._filtered_relations = {} |
| 322 |
|
| 323 |
@property |
| 324 |
def output_field(self): |
| 325 |
if len(self.select) == 1: |
| 326 |
select = self.select[0] |
| 327 |
return getattr(select, "target", None) or select.field |
| 328 |
elif len(self.annotation_select) == 1: |
| 329 |
return next(iter(self.annotation_select.values())).output_field |
| 330 |
|
| 331 |
@cached_property |
| 332 |
def base_table(self): |
| 333 |
for alias in self.alias_map: |
| 334 |
return alias |
| 335 |
|
| 336 |
def __str__(self): |
| 337 |
""" |
| 338 |
Return the query as a string of SQL with the parameter values |
| 339 |
substituted in (use sql_with_params() to see the unsubstituted string). |
| 340 |
|
| 341 |
Parameter values won't necessarily be quoted correctly, since that is |
| 342 |
done by the database interface at execution time. |
| 343 |
""" |
| 344 |
sql, params = self.sql_with_params() |
| 345 |
return sql % params |
| 346 |
|
| 347 |
def sql_with_params(self): |
| 348 |
""" |
| 349 |
Return the query as an SQL string and the parameters that will be |
| 350 |
substituted into the query. |
| 351 |
""" |
| 352 |
return self.get_compiler(DEFAULT_DB_ALIAS).as_sql() |
| 353 |
|
| 354 |
def __deepcopy__(self, memo): |
| 355 |
"""Limit the amount of work when a Query is deepcopied.""" |
| 356 |
result = self.clone() |
| 357 |
memo[id(self)] = result |
| 358 |
return result |
| 359 |
|
| 360 |
def get_compiler(self, using=None, connection=None, elide_empty=True): |
| 361 |
if using is None and connection is None: |
| 362 |
raise ValueError("Need either using or connection") |
| 363 |
if using: |
| 364 |
connection = connections[using] |
| 365 |
return connection.ops.compiler(self.compiler)( |
| 366 |
self, connection, using, elide_empty |
| 367 |
) |
| 368 |
|
| 369 |
def get_meta(self): |
| 370 |
""" |
| 371 |
Return the Options instance (the model._meta) from which to start |
| 372 |
processing. Normally, this is self.model._meta, but it can be changed |
| 373 |
by subclasses. |
| 374 |
""" |
| 375 |
if self.model: |
| 376 |
return self.model._meta |
| 377 |
|
| 378 |
def clone(self): |
| 379 |
""" |
| 380 |
Return a copy of the current Query. A lightweight alternative to |
| 381 |
deepcopy(). |
| 382 |
""" |
| 383 |
obj = Empty() |
| 384 |
obj.__class__ = self.__class__ |
| 385 |
# Copy references to everything. |
| 386 |
obj.__dict__ = self.__dict__.copy() |
| 387 |
# Clone attributes that can't use shallow copy. |
| 388 |
obj.alias_refcount = self.alias_refcount.copy() |
| 389 |
obj.alias_map = self.alias_map.copy() |
| 390 |
obj.external_aliases = self.external_aliases.copy() |
| 391 |
obj.table_map = self.table_map.copy() |
| 392 |
obj.where = self.where.clone() |
| 393 |
obj.annotations = self.annotations.copy() |
| 394 |
if self.annotation_select_mask is not None: |
| 395 |
obj.annotation_select_mask = self.annotation_select_mask.copy() |
| 396 |
if self.combined_queries: |
| 397 |
obj.combined_queries = tuple( |
| 398 |
[query.clone() for query in self.combined_queries] |
| 399 |
) |
| 400 |
# _annotation_select_cache cannot be copied, as doing so breaks the |
| 401 |
# (necessary) state in which both annotations and |
| 402 |
# _annotation_select_cache point to the same underlying objects. |
| 403 |
# It will get re-populated in the cloned queryset the next time it's |
| 404 |
# used. |
| 405 |
obj._annotation_select_cache = None |
| 406 |
obj.extra = self.extra.copy() |
| 407 |
if self.extra_select_mask is not None: |
| 408 |
obj.extra_select_mask = self.extra_select_mask.copy() |
| 409 |
if self._extra_select_cache is not None: |
| 410 |
obj._extra_select_cache = self._extra_select_cache.copy() |
| 411 |
if self.select_related is not False: |
| 412 |
# Use deepcopy because select_related stores fields in nested |
| 413 |
# dicts. |
| 414 |
obj.select_related = copy.deepcopy(obj.select_related) |
| 415 |
if "subq_aliases" in self.__dict__: |
| 416 |
obj.subq_aliases = self.subq_aliases.copy() |
| 417 |
obj.used_aliases = self.used_aliases.copy() |
| 418 |
obj._filtered_relations = self._filtered_relations.copy() |
| 419 |
# Clear the cached_property, if it exists. |
| 420 |
obj.__dict__.pop("base_table", None) |
| 421 |
return obj |
| 422 |
|
| 423 |
def chain(self, klass=None): |
| 424 |
""" |
| 425 |
Return a copy of the current Query that's ready for another operation. |
| 426 |
The klass argument changes the type of the Query, e.g. UpdateQuery. |
| 427 |
""" |
| 428 |
obj = self.clone() |
| 429 |
if klass and obj.__class__ != klass: |
| 430 |
obj.__class__ = klass |
| 431 |
if not obj.filter_is_sticky: |
| 432 |
obj.used_aliases = set() |
| 433 |
obj.filter_is_sticky = False |
| 434 |
if hasattr(obj, "_setup_query"): |
| 435 |
obj._setup_query() |
| 436 |
return obj |
| 437 |
|
| 438 |
def relabeled_clone(self, change_map): |
| 439 |
clone = self.clone() |
| 440 |
clone.change_aliases(change_map) |
| 441 |
return clone |
| 442 |
|
| 443 |
def _get_col(self, target, field, alias): |
| 444 |
if not self.alias_cols: |
| 445 |
alias = None |
| 446 |
return target.get_col(alias, field) |
| 447 |
|
| 448 |
def get_aggregation(self, using, aggregate_exprs): |
| 449 |
""" |
| 450 |
Return the dictionary with the values of the existing aggregations. |
| 451 |
""" |
| 452 |
if not aggregate_exprs: |
| 453 |
return {} |
| 454 |
# Store annotation mask prior to temporarily adding aggregations for |
| 455 |
# resolving purpose to facilitate their subsequent removal. |
| 456 |
refs_subquery = False |
| 457 |
refs_window = False |
| 458 |
replacements = {} |
| 459 |
annotation_select_mask = self.annotation_select_mask |
| 460 |
for alias, aggregate_expr in aggregate_exprs.items(): |
| 461 |
self.check_alias(alias) |
| 462 |
aggregate = aggregate_expr.resolve_expression( |
| 463 |
self, allow_joins=True, reuse=None, summarize=True |
| 464 |
) |
| 465 |
if not aggregate.contains_aggregate: |
| 466 |
raise TypeError("%s is not an aggregate expression" % alias) |
| 467 |
# Temporarily add aggregate to annotations to allow remaining |
| 468 |
# members of `aggregates` to resolve against each others. |
| 469 |
self.append_annotation_mask([alias]) |
| 470 |
aggregate_refs = aggregate.get_refs() |
| 471 |
refs_subquery |= any( |
| 472 |
getattr(self.annotations[ref], "contains_subquery", False) |
| 473 |
for ref in aggregate_refs |
| 474 |
) |
| 475 |
refs_window |= any( |
| 476 |
getattr(self.annotations[ref], "contains_over_clause", True) |
| 477 |
for ref in aggregate_refs |
| 478 |
) |
| 479 |
aggregate = aggregate.replace_expressions(replacements) |
| 480 |
self.annotations[alias] = aggregate |
| 481 |
replacements[Ref(alias, aggregate)] = aggregate |
| 482 |
# Stash resolved aggregates now that they have been allowed to resolve |
| 483 |
# against each other. |
| 484 |
aggregates = {alias: self.annotations.pop(alias) for alias in aggregate_exprs} |
| 485 |
self.set_annotation_mask(annotation_select_mask) |
| 486 |
# Existing usage of aggregation can be determined by the presence of |
| 487 |
# selected aggregates but also by filters against aliased aggregates. |
| 488 |
_, having, qualify = self.where.split_having_qualify() |
| 489 |
has_existing_aggregation = ( |
| 490 |
any( |
| 491 |
getattr(annotation, "contains_aggregate", True) |
| 492 |
for annotation in self.annotations.values() |
| 493 |
) |
| 494 |
or having |
| 495 |
) |
| 496 |
set_returning_annotations = { |
| 497 |
alias |
| 498 |
for alias, annotation in self.annotation_select.items() |
| 499 |
if getattr(annotation, "set_returning", False) |
| 500 |
} |
| 501 |
# Decide if we need to use a subquery. |
| 502 |
# |
| 503 |
# Existing aggregations would cause incorrect results as |
| 504 |
# get_aggregation() must produce just one result and thus must not use |
| 505 |
# GROUP BY. |
| 506 |
# |
| 507 |
# If the query has limit or distinct, or uses set operations, then |
| 508 |
# those operations must be done in a subquery so that the query |
| 509 |
# aggregates on the limit and/or distinct results instead of applying |
| 510 |
# the distinct and limit after the aggregation. |
| 511 |
if ( |
| 512 |
isinstance(self.group_by, tuple) |
| 513 |
or self.is_sliced |
| 514 |
or has_existing_aggregation |
| 515 |
or refs_subquery |
| 516 |
or refs_window |
| 517 |
or qualify |
| 518 |
or self.distinct |
| 519 |
or self.combinator |
| 520 |
or set_returning_annotations |
| 521 |
): |
| 522 |
from django.db.models.sql.subqueries import AggregateQuery |
| 523 |
|
| 524 |
inner_query = self.clone() |
| 525 |
inner_query.subquery = True |
| 526 |
outer_query = AggregateQuery(self.model, inner_query) |
| 527 |
inner_query.select_for_update = False |
| 528 |
inner_query.select_related = False |
| 529 |
inner_query.set_annotation_mask(self.annotation_select) |
| 530 |
# Queries with distinct_fields need ordering and when a limit is |
| 531 |
# applied we must take the slice from the ordered query. Otherwise |
| 532 |
# no need for ordering. |
| 533 |
inner_query.clear_ordering(force=False) |
| 534 |
if not inner_query.distinct: |
| 535 |
# If the inner query uses default select and it has some |
| 536 |
# aggregate annotations, then we must make sure the inner |
| 537 |
# query is grouped by the main model's primary key. However, |
| 538 |
# clearing the select clause can alter results if distinct is |
| 539 |
# used. |
| 540 |
if inner_query.default_cols and has_existing_aggregation: |
| 541 |
inner_query.group_by = ( |
| 542 |
self.model._meta.pk.get_col(inner_query.get_initial_alias()), |
| 543 |
) |
| 544 |
inner_query.default_cols = False |
| 545 |
if not qualify and not self.combinator: |
| 546 |
# Mask existing annotations that are not referenced by |
| 547 |
# aggregates to be pushed to the outer query unless |
| 548 |
# filtering against window functions or if the query is |
| 549 |
# combined as both would require complex realiasing logic. |
| 550 |
annotation_mask = set() |
| 551 |
if isinstance(self.group_by, tuple): |
| 552 |
for expr in self.group_by: |
| 553 |
annotation_mask |= expr.get_refs() |
| 554 |
for aggregate in aggregates.values(): |
| 555 |
annotation_mask |= aggregate.get_refs() |
| 556 |
# Avoid eliding expressions that might have an incidence on |
| 557 |
# the implicit grouping logic. |
| 558 |
for annotation_alias, annotation in self.annotation_select.items(): |
| 559 |
if annotation.get_group_by_cols(): |
| 560 |
annotation_mask.add(annotation_alias) |
| 561 |
inner_query.set_annotation_mask(annotation_mask) |
| 562 |
# Annotations that possibly return multiple rows cannot |
| 563 |
# be masked as they might have an incidence on the query. |
| 564 |
annotation_mask |= set_returning_annotations |
| 565 |
|
| 566 |
# Add aggregates to the outer AggregateQuery. This requires making |
| 567 |
# sure all columns referenced by the aggregates are selected in the |
| 568 |
# inner query. It is achieved by retrieving all column references |
| 569 |
# by the aggregates, explicitly selecting them in the inner query, |
| 570 |
# and making sure the aggregates are repointed to them. |
| 571 |
col_refs = {} |
| 572 |
for alias, aggregate in aggregates.items(): |
| 573 |
replacements = {} |
| 574 |
for col in self._gen_cols([aggregate], resolve_refs=False): |
| 575 |
if not (col_ref := col_refs.get(col)): |
| 576 |
index = len(col_refs) + 1 |
| 577 |
col_alias = f"__col{index}" |
| 578 |
col_ref = Ref(col_alias, col) |
| 579 |
col_refs[col] = col_ref |
| 580 |
inner_query.add_annotation(col, col_alias) |
| 581 |
replacements[col] = col_ref |
| 582 |
outer_query.annotations[alias] = aggregate.replace_expressions( |
| 583 |
replacements |
| 584 |
) |
| 585 |
if ( |
| 586 |
inner_query.select == () |
| 587 |
and not inner_query.default_cols |
| 588 |
and not inner_query.annotation_select_mask |
| 589 |
): |
| 590 |
# In case of Model.objects[0:3].count(), there would be no |
| 591 |
# field selected in the inner query, yet we must use a subquery. |
| 592 |
# So, make sure at least one field is selected. |
| 593 |
inner_query.select = ( |
| 594 |
self.model._meta.pk.get_col(inner_query.get_initial_alias()), |
| 595 |
) |
| 596 |
else: |
| 597 |
outer_query = self |
| 598 |
self.select = () |
| 599 |
self.selected = None |
| 600 |
self.default_cols = False |
| 601 |
self.extra = {} |
| 602 |
if self.annotations: |
| 603 |
# Inline reference to existing annotations and mask them as |
| 604 |
# they are unnecessary given only the summarized aggregations |
| 605 |
# are requested. |
| 606 |
replacements = { |
| 607 |
Ref(alias, annotation): annotation |
| 608 |
for alias, annotation in self.annotations.items() |
| 609 |
} |
| 610 |
self.annotations = { |
| 611 |
alias: aggregate.replace_expressions(replacements) |
| 612 |
for alias, aggregate in aggregates.items() |
| 613 |
} |
| 614 |
else: |
| 615 |
self.annotations = aggregates |
| 616 |
self.set_annotation_mask(aggregates) |
| 617 |
|
| 618 |
empty_set_result = [ |
| 619 |
expression.empty_result_set_value |
| 620 |
for expression in outer_query.annotation_select.values() |
| 621 |
] |
| 622 |
elide_empty = not any(result is NotImplemented for result in empty_set_result) |
| 623 |
outer_query.clear_ordering(force=True) |
| 624 |
outer_query.clear_limits() |
| 625 |
outer_query.select_for_update = False |
| 626 |
outer_query.select_related = False |
| 627 |
compiler = outer_query.get_compiler(using, elide_empty=elide_empty) |
| 628 |
result = compiler.execute_sql(SINGLE) |
| 629 |
if result is None: |
| 630 |
result = empty_set_result |
| 631 |
else: |
| 632 |
cols = outer_query.annotation_select.values() |
| 633 |
converters = compiler.get_converters(cols) |
| 634 |
rows = compiler.apply_converters((result,), converters) |
| 635 |
if compiler.has_composite_fields(cols): |
| 636 |
rows = compiler.composite_fields_to_tuples(rows, cols) |
| 637 |
result = next(rows) |
| 638 |
|
| 639 |
return dict(zip(outer_query.annotation_select, result)) |
| 640 |
|
| 641 |
def get_count(self, using): |
| 642 |
""" |
| 643 |
Perform a COUNT() query using the current filter constraints. |
| 644 |
""" |
| 645 |
obj = self.clone() |
| 646 |
return obj.get_aggregation(using, {"__count": Count("*")})["__count"] |
| 647 |
|
| 648 |
def has_filters(self): |
| 649 |
return self.where |
| 650 |
|
| 651 |
def exists(self, limit=True): |
| 652 |
q = self.clone() |
| 653 |
if not (q.distinct and q.is_sliced): |
| 654 |
if q.group_by is True: |
| 655 |
q.add_fields( |
| 656 |
(f.attname for f in self.model._meta.concrete_fields), False |
| 657 |
) |
| 658 |
# Disable GROUP BY aliases to avoid orphaning references to the |
| 659 |
# SELECT clause which is about to be cleared. |
| 660 |
q.set_group_by(allow_aliases=False) |
| 661 |
q.clear_select_clause() |
| 662 |
if q.combined_queries and q.combinator == "union": |
| 663 |
q.combined_queries = tuple( |
| 664 |
combined_query.exists(limit=False) |
| 665 |
for combined_query in q.combined_queries |
| 666 |
) |
| 667 |
q.clear_ordering(force=True) |
| 668 |
if limit is True: |
| 669 |
q.set_limits(high=1) |
| 670 |
q.add_annotation(Value(1), "a") |
| 671 |
return q |
| 672 |
|
| 673 |
def has_results(self, using): |
| 674 |
q = self.exists() |
| 675 |
compiler = q.get_compiler(using=using) |
| 676 |
return compiler.has_results() |
| 677 |
|
| 678 |
def explain(self, using, format=None, **options): |
| 679 |
q = self.clone() |
| 680 |
for option_name in options: |
| 681 |
if ( |
| 682 |
not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name) |
| 683 |
or "--" in option_name |
| 684 |
): |
| 685 |
raise ValueError(f"Invalid option name: {option_name!r}.") |
| 686 |
q.explain_info = ExplainInfo(format, options) |
| 687 |
compiler = q.get_compiler(using=using) |
| 688 |
return "\n".join(compiler.explain_query()) |
| 689 |
|
| 690 |
def combine(self, rhs, connector): |
| 691 |
""" |
| 692 |
Merge the 'rhs' query into the current one (with any 'rhs' effects |
| 693 |
being applied *after* (that is, "to the right of") anything in the |
| 694 |
current query. 'rhs' is not modified during a call to this function. |
| 695 |
|
| 696 |
The 'connector' parameter describes how to connect filters from the |
| 697 |
'rhs' query. |
| 698 |
""" |
| 699 |
if self.model != rhs.model: |
| 700 |
raise TypeError("Cannot combine queries on two different base models.") |
| 701 |
if self.is_sliced: |
| 702 |
raise TypeError("Cannot combine queries once a slice has been taken.") |
| 703 |
if self.distinct != rhs.distinct: |
| 704 |
raise TypeError("Cannot combine a unique query with a non-unique query.") |
| 705 |
if self.distinct_fields != rhs.distinct_fields: |
| 706 |
raise TypeError("Cannot combine queries with different distinct fields.") |
| 707 |
|
| 708 |
# If lhs and rhs shares the same alias prefix, it is possible to have |
| 709 |
# conflicting alias changes like T4 -> T5, T5 -> T6, which might end up |
| 710 |
# as T4 -> T6 while combining two querysets. To prevent this, change an |
| 711 |
# alias prefix of the rhs and update current aliases accordingly, |
| 712 |
# except if the alias is the base table since it must be present in the |
| 713 |
# query on both sides. |
| 714 |
initial_alias = self.get_initial_alias() |
| 715 |
rhs = rhs.clone() |
| 716 |
rhs.bump_prefix(self, exclude={initial_alias}) |
| 717 |
|
| 718 |
# Work out how to relabel the rhs aliases, if necessary. |
| 719 |
change_map = {} |
| 720 |
conjunction = connector == AND |
| 721 |
|
| 722 |
# Determine which existing joins can be reused. When combining the |
| 723 |
# query with AND we must recreate all joins for m2m filters. When |
| 724 |
# combining with OR we can reuse joins. The reason is that in AND |
| 725 |
# case a single row can't fulfill a condition like: |
| 726 |
# revrel__col=1 & revrel__col=2 |
| 727 |
# But, there might be two different related rows matching this |
| 728 |
# condition. In OR case a single True is enough, so single row is |
| 729 |
# enough, too. |
| 730 |
# |
| 731 |
# Note that we will be creating duplicate joins for non-m2m joins in |
| 732 |
# the AND case. The results will be correct but this creates too many |
| 733 |
# joins. This is something that could be fixed later on. |
| 734 |
reuse = set() if conjunction else set(self.alias_map) |
| 735 |
joinpromoter = JoinPromoter(connector, 2, False) |
| 736 |
joinpromoter.add_votes( |
| 737 |
j for j in self.alias_map if self.alias_map[j].join_type == INNER |
| 738 |
) |
| 739 |
rhs_votes = set() |
| 740 |
# Now, add the joins from rhs query into the new query (skipping base |
| 741 |
# table). |
| 742 |
rhs_tables = list(rhs.alias_map)[1:] |
| 743 |
for alias in rhs_tables: |
| 744 |
join = rhs.alias_map[alias] |
| 745 |
# If the left side of the join was already relabeled, use the |
| 746 |
# updated alias. |
| 747 |
join = join.relabeled_clone(change_map) |
| 748 |
new_alias = self.join(join, reuse=reuse) |
| 749 |
if join.join_type == INNER: |
| 750 |
rhs_votes.add(new_alias) |
| 751 |
# We can't reuse the same join again in the query. If we have two |
| 752 |
# distinct joins for the same connection in rhs query, then the |
| 753 |
# combined query must have two joins, too. |
| 754 |
reuse.discard(new_alias) |
| 755 |
if alias != new_alias: |
| 756 |
change_map[alias] = new_alias |
| 757 |
if not rhs.alias_refcount[alias]: |
| 758 |
# The alias was unused in the rhs query. Unref it so that it |
| 759 |
# will be unused in the new query, too. We have to add and |
| 760 |
# unref the alias so that join promotion has information of |
| 761 |
# the join type for the unused alias. |
| 762 |
self.unref_alias(new_alias) |
| 763 |
joinpromoter.add_votes(rhs_votes) |
| 764 |
joinpromoter.update_join_types(self) |
| 765 |
|
| 766 |
# Combine subqueries aliases to ensure aliases relabelling properly |
| 767 |
# handle subqueries when combining where and select clauses. |
| 768 |
self.subq_aliases |= rhs.subq_aliases |
| 769 |
|
| 770 |
# Now relabel a copy of the rhs where-clause and add it to the current |
| 771 |
# one. |
| 772 |
w = rhs.where.clone() |
| 773 |
w.relabel_aliases(change_map) |
| 774 |
self.where.add(w, connector) |
| 775 |
|
| 776 |
# Selection columns and extra extensions are those provided by 'rhs'. |
| 777 |
if rhs.select: |
| 778 |
self.set_select([col.relabeled_clone(change_map) for col in rhs.select]) |
| 779 |
else: |
| 780 |
self.select = () |
| 781 |
|
| 782 |
if connector == OR: |
| 783 |
# It would be nice to be able to handle this, but the queries don't |
| 784 |
# really make sense (or return consistent value sets). Not worth |
| 785 |
# the extra complexity when you can write a real query instead. |
| 786 |
if self.extra and rhs.extra: |
| 787 |
raise ValueError( |
| 788 |
"When merging querysets using 'or', you cannot have " |
| 789 |
"extra(select=...) on both sides." |
| 790 |
) |
| 791 |
self.extra.update(rhs.extra) |
| 792 |
extra_select_mask = set() |
| 793 |
if self.extra_select_mask is not None: |
| 794 |
extra_select_mask.update(self.extra_select_mask) |
| 795 |
if rhs.extra_select_mask is not None: |
| 796 |
extra_select_mask.update(rhs.extra_select_mask) |
| 797 |
if extra_select_mask: |
| 798 |
self.set_extra_mask(extra_select_mask) |
| 799 |
self.extra_tables += rhs.extra_tables |
| 800 |
|
| 801 |
# Ordering uses the 'rhs' ordering, unless it has none, in which case |
| 802 |
# the current ordering is used. |
| 803 |
self.order_by = rhs.order_by or self.order_by |
| 804 |
self.extra_order_by = rhs.extra_order_by or self.extra_order_by |
| 805 |
|
| 806 |
def _get_defer_select_mask(self, opts, mask, select_mask=None): |
| 807 |
if select_mask is None: |
| 808 |
select_mask = {} |
| 809 |
select_mask[opts.pk] = {} |
| 810 |
# All concrete fields and related objects that are not part of the |
| 811 |
# defer mask must be included. If a relational field is encountered it |
| 812 |
# gets added to the mask for it be considered if `select_related` and |
| 813 |
# the cycle continues by recursively calling this function. |
| 814 |
for field in opts.concrete_fields + opts.related_objects: |
| 815 |
field_mask = mask.pop(field.name, None) |
| 816 |
field_att_mask = None |
| 817 |
if field_attname := getattr(field, "attname", None): |
| 818 |
field_att_mask = mask.pop(field_attname, None) |
| 819 |
if field_mask is None and field_att_mask is None: |
| 820 |
select_mask.setdefault(field, {}) |
| 821 |
elif field_mask: |
| 822 |
if not field.is_relation: |
| 823 |
raise FieldError(next(iter(field_mask))) |
| 824 |
# Virtual fields such as many-to-many and generic foreign keys |
| 825 |
# cannot be effectively deferred. Historically, they were |
| 826 |
# allowed to be passed to QuerySet.defer(). Ignore such field |
| 827 |
# references until a layer of validation at mask alteration |
| 828 |
# time is eventually implemented. |
| 829 |
if field.many_to_many: |
| 830 |
continue |
| 831 |
field_select_mask = select_mask.setdefault(field, {}) |
| 832 |
related_model = field.related_model._meta.concrete_model |
| 833 |
self._get_defer_select_mask( |
| 834 |
related_model._meta, field_mask, field_select_mask |
| 835 |
) |
| 836 |
# Remaining defer entries must be references to filtered relations |
| 837 |
# otherwise they are surfaced as missing field errors. |
| 838 |
for field_name, field_mask in mask.items(): |
| 839 |
if filtered_relation := self._filtered_relations.get(field_name): |
| 840 |
relation = opts.get_field(filtered_relation.relation_name) |
| 841 |
field_select_mask = select_mask.setdefault((field_name, relation), {}) |
| 842 |
related_model = relation.related_model._meta.concrete_model |
| 843 |
self._get_defer_select_mask( |
| 844 |
related_model._meta, field_mask, field_select_mask |
| 845 |
) |
| 846 |
else: |
| 847 |
opts.get_field(field_name) |
| 848 |
return select_mask |
| 849 |
|
| 850 |
def _get_only_select_mask(self, opts, mask, select_mask=None): |
| 851 |
if select_mask is None: |
| 852 |
select_mask = {} |
| 853 |
select_mask[opts.pk] = {} |
| 854 |
# Only include fields mentioned in the mask. |
| 855 |
for field_name, field_mask in mask.items(): |
| 856 |
field = opts.get_field(field_name) |
| 857 |
field_select_mask = select_mask.setdefault(field, {}) |
| 858 |
if field_mask: |
| 859 |
if not field.is_relation: |
| 860 |
raise FieldError(next(iter(field_mask))) |
| 861 |
related_model = field.remote_field.model._meta.concrete_model |
| 862 |
self._get_only_select_mask( |
| 863 |
related_model._meta, field_mask, field_select_mask |
| 864 |
) |
| 865 |
return select_mask |
| 866 |
|
| 867 |
def get_select_mask(self): |
| 868 |
""" |
| 869 |
Convert the self.deferred_loading data structure to an alternate data |
| 870 |
structure, describing the field that *will* be loaded. This is used to |
| 871 |
compute the columns to select from the database and also by the |
| 872 |
QuerySet class to work out which fields are being initialized on each |
| 873 |
model. Models that have all their fields included aren't mentioned in |
| 874 |
the result, only those that have field restrictions in place. |
| 875 |
""" |
| 876 |
field_names, defer = self.deferred_loading |
| 877 |
if not field_names: |
| 878 |
return {} |
| 879 |
mask = {} |
| 880 |
for field_name in field_names: |
| 881 |
part_mask = mask |
| 882 |
for part in field_name.split(LOOKUP_SEP): |
| 883 |
part_mask = part_mask.setdefault(part, {}) |
| 884 |
opts = self.get_meta() |
| 885 |
if defer: |
| 886 |
return self._get_defer_select_mask(opts, mask) |
| 887 |
return self._get_only_select_mask(opts, mask) |
| 888 |
|
| 889 |
def table_alias(self, table_name, create=False, filtered_relation=None): |
| 890 |
""" |
| 891 |
Return a table alias for the given table_name and whether this is a |
| 892 |
new alias or not. |
| 893 |
|
| 894 |
If 'create' is true, a new alias is always created. Otherwise, the |
| 895 |
most recently created alias for the table (if one exists) is reused. |
| 896 |
""" |
| 897 |
alias_list = self.table_map.get(table_name) |
| 898 |
if not create and alias_list: |
| 899 |
alias = alias_list[0] |
| 900 |
self.alias_refcount[alias] += 1 |
| 901 |
return alias, False |
| 902 |
|
| 903 |
# Create a new alias for this table. |
| 904 |
if alias_list: |
| 905 |
alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1) |
| 906 |
alias_list.append(alias) |
| 907 |
else: |
| 908 |
# The first occurrence of a table uses the table name directly. |
| 909 |
alias = ( |
| 910 |
filtered_relation.alias if filtered_relation is not None else table_name |
| 911 |
) |
| 912 |
self.table_map[table_name] = [alias] |
| 913 |
self.alias_refcount[alias] = 1 |
| 914 |
return alias, True |
| 915 |
|
| 916 |
def ref_alias(self, alias): |
| 917 |
"""Increases the reference count for this alias.""" |
| 918 |
self.alias_refcount[alias] += 1 |
| 919 |
|
| 920 |
def unref_alias(self, alias, amount=1): |
| 921 |
"""Decreases the reference count for this alias.""" |
| 922 |
self.alias_refcount[alias] -= amount |
| 923 |
|
| 924 |
def promote_joins(self, aliases): |
| 925 |
""" |
| 926 |
Promote recursively the join type of given aliases and its children to |
| 927 |
an outer join. If 'unconditional' is False, only promote the join if |
| 928 |
it is nullable or the parent join is an outer join. |
| 929 |
|
| 930 |
The children promotion is done to avoid join chains that contain a LOUTER |
| 931 |
b INNER c. So, if we have currently a INNER b INNER c and a->b is promoted, |
| 932 |
then we must also promote b->c automatically, or otherwise the promotion |
| 933 |
of a->b doesn't actually change anything in the query results. |
| 934 |
""" |
| 935 |
aliases = list(aliases) |
| 936 |
while aliases: |
| 937 |
alias = aliases.pop(0) |
| 938 |
if self.alias_map[alias].join_type is None: |
| 939 |
# This is the base table (first FROM entry) - this table |
| 940 |
# isn't really joined at all in the query, so we should not |
| 941 |
# alter its join type. |
| 942 |
continue |
| 943 |
# Only the first alias (skipped above) should have None join_type |
| 944 |
assert self.alias_map[alias].join_type is not None |
| 945 |
parent_alias = self.alias_map[alias].parent_alias |
| 946 |
parent_louter = ( |
| 947 |
parent_alias and self.alias_map[parent_alias].join_type == LOUTER |
| 948 |
) |
| 949 |
already_louter = self.alias_map[alias].join_type == LOUTER |
| 950 |
if (self.alias_map[alias].nullable or parent_louter) and not already_louter: |
| 951 |
self.alias_map[alias] = self.alias_map[alias].promote() |
| 952 |
# Join type of 'alias' changed, so re-examine all aliases that |
| 953 |
# refer to this one. |
| 954 |
aliases.extend( |
| 955 |
join |
| 956 |
for join in self.alias_map |
| 957 |
if self.alias_map[join].parent_alias == alias |
| 958 |
and join not in aliases |
| 959 |
) |
| 960 |
|
| 961 |
def demote_joins(self, aliases): |
| 962 |
""" |
| 963 |
Change join type from LOUTER to INNER for all joins in aliases. |
| 964 |
|
| 965 |
Similarly to promote_joins(), this method must ensure no join chains |
| 966 |
containing first an outer, then an inner join are generated. If we |
| 967 |
are demoting b->c join in chain a LOUTER b LOUTER c then we must |
| 968 |
demote a->b automatically, or otherwise the demotion of b->c doesn't |
| 969 |
actually change anything in the query results. . |
| 970 |
""" |
| 971 |
aliases = list(aliases) |
| 972 |
while aliases: |
| 973 |
alias = aliases.pop(0) |
| 974 |
if self.alias_map[alias].join_type == LOUTER: |
| 975 |
self.alias_map[alias] = self.alias_map[alias].demote() |
| 976 |
parent_alias = self.alias_map[alias].parent_alias |
| 977 |
if self.alias_map[parent_alias].join_type == INNER: |
| 978 |
aliases.append(parent_alias) |
| 979 |
|
| 980 |
def reset_refcounts(self, to_counts): |
| 981 |
""" |
| 982 |
Reset reference counts for aliases so that they match the value passed |
| 983 |
in `to_counts`. |
| 984 |
""" |
| 985 |
for alias, cur_refcount in self.alias_refcount.copy().items(): |
| 986 |
unref_amount = cur_refcount - to_counts.get(alias, 0) |
| 987 |
self.unref_alias(alias, unref_amount) |
| 988 |
|
| 989 |
def change_aliases(self, change_map): |
| 990 |
""" |
| 991 |
Change the aliases in change_map (which maps old-alias -> new-alias), |
| 992 |
relabelling any references to them in select columns and the where |
| 993 |
clause. |
| 994 |
""" |
| 995 |
if not change_map: |
| 996 |
return self |
| 997 |
# If keys and values of change_map were to intersect, an alias might be |
| 998 |
# updated twice (e.g. T4 -> T5, T5 -> T6, so also T4 -> T6) depending |
| 999 |
# on their order in change_map. |
| 1000 |
assert set(change_map).isdisjoint(change_map.values()) |
| 1001 |
|
| 1002 |
# 1. Update references in "select" (normal columns plus aliases), |
| 1003 |
# "group by" and "where". |
| 1004 |
self.where.relabel_aliases(change_map) |
| 1005 |
if isinstance(self.group_by, tuple): |
| 1006 |
self.group_by = tuple( |
| 1007 |
[col.relabeled_clone(change_map) for col in self.group_by] |
| 1008 |
) |
| 1009 |
self.select = tuple([col.relabeled_clone(change_map) for col in self.select]) |
| 1010 |
self.annotations = self.annotations and { |
| 1011 |
key: col.relabeled_clone(change_map) |
| 1012 |
for key, col in self.annotations.items() |
| 1013 |
} |
| 1014 |
|
| 1015 |
# 2. Rename the alias in the internal table/alias datastructures. |
| 1016 |
for old_alias, new_alias in change_map.items(): |
| 1017 |
if old_alias not in self.alias_map: |
| 1018 |
continue |
| 1019 |
alias_data = self.alias_map[old_alias].relabeled_clone(change_map) |
| 1020 |
self.alias_map[new_alias] = alias_data |
| 1021 |
self.alias_refcount[new_alias] = self.alias_refcount[old_alias] |
| 1022 |
del self.alias_refcount[old_alias] |
| 1023 |
del self.alias_map[old_alias] |
| 1024 |
|
| 1025 |
table_aliases = self.table_map[alias_data.table_name] |
| 1026 |
for pos, alias in enumerate(table_aliases): |
| 1027 |
if alias == old_alias: |
| 1028 |
table_aliases[pos] = new_alias |
| 1029 |
break |
| 1030 |
|
| 1031 |
# 3. Rename the direct external aliases and the ones of combined |
| 1032 |
# queries (union, intersection, difference). |
| 1033 |
self.external_aliases = { |
| 1034 |
# Table is aliased or it's being changed and thus is aliased. |
| 1035 |
change_map.get(alias, alias): (aliased or alias in change_map) |
| 1036 |
for alias, aliased in self.external_aliases.items() |
| 1037 |
} |
| 1038 |
for combined_query in self.combined_queries: |
| 1039 |
external_change_map = { |
| 1040 |
alias: aliased |
| 1041 |
for alias, aliased in change_map.items() |
| 1042 |
if alias in combined_query.external_aliases |
| 1043 |
} |
| 1044 |
combined_query.change_aliases(external_change_map) |
| 1045 |
|
| 1046 |
def bump_prefix(self, other_query, exclude=None): |
| 1047 |
""" |
| 1048 |
Change the alias prefix to the next letter in the alphabet in a way |
| 1049 |
that the other query's aliases and this query's aliases will not |
| 1050 |
conflict. Even tables that previously had no alias will get an alias |
| 1051 |
after this call. To prevent changing aliases use the exclude parameter. |
| 1052 |
""" |
| 1053 |
|
| 1054 |
def prefix_gen(): |
| 1055 |
""" |
| 1056 |
Generate a sequence of characters in alphabetical order: |
| 1057 |
-> 'A', 'B', 'C', ... |
| 1058 |
|
| 1059 |
When the alphabet is finished, the sequence will continue with the |
| 1060 |
Cartesian product: |
| 1061 |
-> 'AA', 'AB', 'AC', ... |
| 1062 |
""" |
| 1063 |
alphabet = ascii_uppercase |
| 1064 |
prefix = chr(ord(self.alias_prefix) + 1) |
| 1065 |
yield prefix |
| 1066 |
for n in count(1): |
| 1067 |
seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet |
| 1068 |
for s in product(seq, repeat=n): |
| 1069 |
yield "".join(s) |
| 1070 |
prefix = None |
| 1071 |
|
| 1072 |
if self.alias_prefix != other_query.alias_prefix: |
| 1073 |
# No clashes between self and outer query should be possible. |
| 1074 |
return |
| 1075 |
|
| 1076 |
# Explicitly avoid infinite loop. The constant divider is based on how |
| 1077 |
# much depth recursive subquery references add to the stack. This value |
| 1078 |
# might need to be adjusted when adding or removing function calls from |
| 1079 |
# the code path in charge of performing these operations. |
| 1080 |
local_recursion_limit = sys.getrecursionlimit() // 16 |
| 1081 |
for pos, prefix in enumerate(prefix_gen()): |
| 1082 |
if prefix not in self.subq_aliases: |
| 1083 |
self.alias_prefix = prefix |
| 1084 |
break |
| 1085 |
if pos > local_recursion_limit: |
| 1086 |
raise RecursionError( |
| 1087 |
"Maximum recursion depth exceeded: too many subqueries." |
| 1088 |
) |
| 1089 |
self.subq_aliases = self.subq_aliases.union([self.alias_prefix]) |
| 1090 |
other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases) |
| 1091 |
if exclude is None: |
| 1092 |
exclude = {} |
| 1093 |
self.change_aliases( |
| 1094 |
{ |
| 1095 |
alias: "%s%d" % (self.alias_prefix, pos) |
| 1096 |
for pos, alias in enumerate(self.alias_map) |
| 1097 |
if alias not in exclude |
| 1098 |
} |
| 1099 |
) |
| 1100 |
|
| 1101 |
def get_initial_alias(self): |
| 1102 |
""" |
| 1103 |
Return the first alias for this query, after increasing its reference |
| 1104 |
count. |
| 1105 |
""" |
| 1106 |
if self.alias_map: |
| 1107 |
alias = self.base_table |
| 1108 |
self.ref_alias(alias) |
| 1109 |
elif self.model: |
| 1110 |
alias = self.join(self.base_table_class(self.get_meta().db_table, None)) |
| 1111 |
else: |
| 1112 |
alias = None |
| 1113 |
return alias |
| 1114 |
|
| 1115 |
def count_active_tables(self): |
| 1116 |
""" |
| 1117 |
Return the number of tables in this query with a non-zero reference |
| 1118 |
count. After execution, the reference counts are zeroed, so tables |
| 1119 |
added in compiler will not be seen by this method. |
| 1120 |
""" |
| 1121 |
return len([1 for count in self.alias_refcount.values() if count]) |
| 1122 |
|
| 1123 |
def join(self, join, reuse=None): |
| 1124 |
""" |
| 1125 |
Return an alias for the 'join', either reusing an existing alias for |
| 1126 |
that join or creating a new one. 'join' is either a base_table_class or |
| 1127 |
join_class. |
| 1128 |
|
| 1129 |
The 'reuse' parameter can be either None which means all joins are |
| 1130 |
reusable, or it can be a set containing the aliases that can be reused. |
| 1131 |
|
| 1132 |
A join is always created as LOUTER if the lhs alias is LOUTER to make |
| 1133 |
sure chains like t1 LOUTER t2 INNER t3 aren't generated. All new |
| 1134 |
joins are created as LOUTER if the join is nullable. |
| 1135 |
""" |
| 1136 |
reuse_aliases = [ |
| 1137 |
a |
| 1138 |
for a, j in self.alias_map.items() |
| 1139 |
if (reuse is None or a in reuse) and j == join |
| 1140 |
] |
| 1141 |
if reuse_aliases: |
| 1142 |
if join.table_alias in reuse_aliases: |
| 1143 |
reuse_alias = join.table_alias |
| 1144 |
else: |
| 1145 |
# Reuse the most recent alias of the joined table |
| 1146 |
# (a many-to-many relation may be joined multiple times). |
| 1147 |
reuse_alias = reuse_aliases[-1] |
| 1148 |
self.ref_alias(reuse_alias) |
| 1149 |
return reuse_alias |
| 1150 |
|
| 1151 |
# No reuse is possible, so we need a new alias. |
| 1152 |
alias, _ = self.table_alias( |
| 1153 |
join.table_name, create=True, filtered_relation=join.filtered_relation |
| 1154 |
) |
| 1155 |
if join.join_type: |
| 1156 |
if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable: |
| 1157 |
join_type = LOUTER |
| 1158 |
else: |
| 1159 |
join_type = INNER |
| 1160 |
join.join_type = join_type |
| 1161 |
join.table_alias = alias |
| 1162 |
self.alias_map[alias] = join |
| 1163 |
if filtered_relation := join.filtered_relation: |
| 1164 |
resolve_reuse = reuse |
| 1165 |
if resolve_reuse is not None: |
| 1166 |
resolve_reuse = set(reuse) | {alias} |
| 1167 |
joins_len = len(self.alias_map) |
| 1168 |
join.filtered_relation = filtered_relation.resolve_expression( |
| 1169 |
self, reuse=resolve_reuse |
| 1170 |
) |
| 1171 |
# Some joins were during expression resolving, they must be present |
| 1172 |
# before the one we just added. |
| 1173 |
if joins_len < len(self.alias_map): |
| 1174 |
self.alias_map[alias] = self.alias_map.pop(alias) |
| 1175 |
return alias |
| 1176 |
|
| 1177 |
def join_parent_model(self, opts, model, alias, seen): |
| 1178 |
""" |
| 1179 |
Make sure the given 'model' is joined in the query. If 'model' isn't |
| 1180 |
a parent of 'opts' or if it is None this method is a no-op. |
| 1181 |
|
| 1182 |
The 'alias' is the root alias for starting the join, 'seen' is a dict |
| 1183 |
of model -> alias of existing joins. It must also contain a mapping |
| 1184 |
of None -> some alias. This will be returned in the no-op case. |
| 1185 |
""" |
| 1186 |
if model in seen: |
| 1187 |
return seen[model] |
| 1188 |
chain = opts.get_base_chain(model) |
| 1189 |
if not chain: |
| 1190 |
return alias |
| 1191 |
curr_opts = opts |
| 1192 |
for int_model in chain: |
| 1193 |
if int_model in seen: |
| 1194 |
curr_opts = int_model._meta |
| 1195 |
alias = seen[int_model] |
| 1196 |
continue |
| 1197 |
# Proxy model have elements in base chain |
| 1198 |
# with no parents, assign the new options |
| 1199 |
# object and skip to the next base in that |
| 1200 |
# case |
| 1201 |
if not curr_opts.parents[int_model]: |
| 1202 |
curr_opts = int_model._meta |
| 1203 |
continue |
| 1204 |
link_field = curr_opts.get_ancestor_link(int_model) |
| 1205 |
join_info = self.setup_joins([link_field.name], curr_opts, alias) |
| 1206 |
curr_opts = int_model._meta |
| 1207 |
alias = seen[int_model] = join_info.joins[-1] |
| 1208 |
return alias or seen[None] |
| 1209 |
|
| 1210 |
def check_alias(self, alias): |
| 1211 |
if FORBIDDEN_ALIAS_PATTERN.search(alias): |
| 1212 |
raise ValueError( |
| 1213 |
"Column aliases cannot contain whitespace characters, hashes, " |
| 1214 |
"control characters, quotation marks, semicolons, or SQL comments." |
| 1215 |
) |
| 1216 |
|
| 1217 |
def add_annotation(self, annotation, alias, select=True): |
| 1218 |
"""Add a single annotation expression to the Query.""" |
| 1219 |
self.check_alias(alias) |
| 1220 |
annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None) |
| 1221 |
if select: |
| 1222 |
self.append_annotation_mask([alias]) |
| 1223 |
else: |
| 1224 |
self.set_annotation_mask(set(self.annotation_select).difference({alias})) |
| 1225 |
self.annotations[alias] = annotation |
| 1226 |
if select and self.selected: |
| 1227 |
self.selected[alias] = alias |
| 1228 |
|
| 1229 |
@property |
| 1230 |
def _subquery_fields_len(self): |
| 1231 |
if self.has_select_fields: |
| 1232 |
return sum( |
| 1233 |
len(self.model._meta.pk_fields) if field == "pk" else 1 |
| 1234 |
for field in self.selected |
| 1235 |
) |
| 1236 |
return len(self.model._meta.pk_fields) |
| 1237 |
|
| 1238 |
def resolve_expression(self, query, *args, **kwargs): |
| 1239 |
clone = self.clone() |
| 1240 |
# Subqueries need to use a different set of aliases than the outer query. |
| 1241 |
clone.bump_prefix(query) |
| 1242 |
clone.subquery = True |
| 1243 |
clone.where.resolve_expression(query, *args, **kwargs) |
| 1244 |
# Resolve combined queries. |
| 1245 |
if clone.combinator: |
| 1246 |
clone.combined_queries = tuple( |
| 1247 |
[ |
| 1248 |
combined_query.resolve_expression(query, *args, **kwargs) |
| 1249 |
for combined_query in clone.combined_queries |
| 1250 |
] |
| 1251 |
) |
| 1252 |
for key, value in clone.annotations.items(): |
| 1253 |
resolved = value.resolve_expression(query, *args, **kwargs) |
| 1254 |
if hasattr(resolved, "external_aliases"): |
| 1255 |
resolved.external_aliases.update(clone.external_aliases) |
| 1256 |
clone.annotations[key] = resolved |
| 1257 |
# Outer query's aliases are considered external. |
| 1258 |
for alias, table in query.alias_map.items(): |
| 1259 |
clone.external_aliases[alias] = ( |
| 1260 |
isinstance(table, Join) |
| 1261 |
and table.join_field.related_model._meta.db_table != alias |
| 1262 |
) or ( |
| 1263 |
isinstance(table, BaseTable) and table.table_name != table.table_alias |
| 1264 |
) |
| 1265 |
return clone |
| 1266 |
|
| 1267 |
def get_external_cols(self): |
| 1268 |
exprs = chain(self.annotations.values(), self.where.children) |
| 1269 |
return [ |
| 1270 |
col |
| 1271 |
for col in self._gen_cols(exprs, include_external=True) |
| 1272 |
if col.alias in self.external_aliases |
| 1273 |
] |
| 1274 |
|
| 1275 |
def get_group_by_cols(self, wrapper=None): |
| 1276 |
# If wrapper is referenced by an alias for an explicit GROUP BY through |
| 1277 |
# values() a reference to this expression and not the self must be |
| 1278 |
# returned to ensure external column references are not grouped against |
| 1279 |
# as well. |
| 1280 |
external_cols = self.get_external_cols() |
| 1281 |
if any(col.possibly_multivalued for col in external_cols): |
| 1282 |
return [wrapper or self] |
| 1283 |
return external_cols |
| 1284 |
|
| 1285 |
def as_sql(self, compiler, connection): |
| 1286 |
# Some backends (e.g. Oracle) raise an error when a subquery contains |
| 1287 |
# unnecessary ORDER BY clause. |
| 1288 |
if ( |
| 1289 |
self.subquery |
| 1290 |
and not connection.features.ignores_unnecessary_order_by_in_subqueries |
| 1291 |
): |
| 1292 |
self.clear_ordering(force=False) |
| 1293 |
for query in self.combined_queries: |
| 1294 |
query.clear_ordering(force=False) |
| 1295 |
sql, params = self.get_compiler(connection=connection).as_sql() |
| 1296 |
if self.subquery: |
| 1297 |
sql = "(%s)" % sql |
| 1298 |
return sql, params |
| 1299 |
|
| 1300 |
def resolve_lookup_value(self, value, can_reuse, allow_joins, summarize=False): |
| 1301 |
if hasattr(value, "resolve_expression"): |
| 1302 |
value = value.resolve_expression( |
| 1303 |
self, |
| 1304 |
reuse=can_reuse, |
| 1305 |
allow_joins=allow_joins, |
| 1306 |
summarize=summarize, |
| 1307 |
) |
| 1308 |
elif isinstance(value, (list, tuple)): |
| 1309 |
# The items of the iterable may be expressions and therefore need |
| 1310 |
# to be resolved independently. |
| 1311 |
values = ( |
| 1312 |
self.resolve_lookup_value(sub_value, can_reuse, allow_joins, summarize) |
| 1313 |
for sub_value in value |
| 1314 |
) |
| 1315 |
type_ = type(value) |
| 1316 |
if hasattr(type_, "_make"): # namedtuple |
| 1317 |
return type_(*values) |
| 1318 |
return type_(values) |
| 1319 |
return value |
| 1320 |
|
| 1321 |
def solve_lookup_type(self, lookup, summarize=False): |
| 1322 |
""" |
| 1323 |
Solve the lookup type from the lookup (e.g.: 'foobar__id__icontains'). |
| 1324 |
""" |
| 1325 |
lookup_splitted = lookup.split(LOOKUP_SEP) |
| 1326 |
if self.annotations: |
| 1327 |
annotation, expression_lookups = refs_expression( |
| 1328 |
lookup_splitted, self.annotations |
| 1329 |
) |
| 1330 |
if annotation: |
| 1331 |
expression = self.annotations[annotation] |
| 1332 |
if summarize: |
| 1333 |
expression = Ref(annotation, expression) |
| 1334 |
return expression_lookups, (), expression |
| 1335 |
_, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta()) |
| 1336 |
field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)] |
| 1337 |
if len(lookup_parts) > 1 and not field_parts: |
| 1338 |
raise FieldError( |
| 1339 |
'Invalid lookup "%s" for model %s".' |
| 1340 |
% (lookup, self.get_meta().model.__name__) |
| 1341 |
) |
| 1342 |
return lookup_parts, field_parts, False |
| 1343 |
|
| 1344 |
def check_query_object_type(self, value, opts, field): |
| 1345 |
""" |
| 1346 |
Check whether the object passed while querying is of the correct type. |
| 1347 |
If not, raise a ValueError specifying the wrong object. |
| 1348 |
""" |
| 1349 |
if hasattr(value, "_meta"): |
| 1350 |
if not check_rel_lookup_compatibility(value._meta.model, opts, field): |
| 1351 |
raise ValueError( |
| 1352 |
'Cannot query "%s": Must be "%s" instance.' |
| 1353 |
% (value, opts.object_name) |
| 1354 |
) |
| 1355 |
|
| 1356 |
def check_related_objects(self, field, value, opts): |
| 1357 |
"""Check the type of object passed to query relations.""" |
| 1358 |
if field.is_relation: |
| 1359 |
# Check that the field and the queryset use the same model in a |
| 1360 |
# query like .filter(author=Author.objects.all()). For example, the |
| 1361 |
# opts would be Author's (from the author field) and value.model |
| 1362 |
# would be Author.objects.all() queryset's .model (Author also). |
| 1363 |
# The field is the related field on the lhs side. |
| 1364 |
if ( |
| 1365 |
isinstance(value, Query) |
| 1366 |
and not value.has_select_fields |
| 1367 |
and not check_rel_lookup_compatibility(value.model, opts, field) |
| 1368 |
): |
| 1369 |
raise ValueError( |
| 1370 |
'Cannot use QuerySet for "%s": Use a QuerySet for "%s".' |
| 1371 |
% (value.model._meta.object_name, opts.object_name) |
| 1372 |
) |
| 1373 |
elif hasattr(value, "_meta"): |
| 1374 |
self.check_query_object_type(value, opts, field) |
| 1375 |
elif hasattr(value, "__iter__"): |
| 1376 |
for v in value: |
| 1377 |
self.check_query_object_type(v, opts, field) |
| 1378 |
|
| 1379 |
def check_filterable(self, expression): |
| 1380 |
"""Raise an error if expression cannot be used in a WHERE clause.""" |
| 1381 |
if hasattr(expression, "resolve_expression") and not getattr( |
| 1382 |
expression, "filterable", True |
| 1383 |
): |
| 1384 |
raise NotSupportedError( |
| 1385 |
expression.__class__.__name__ + " is disallowed in the filter " |
| 1386 |
"clause." |
| 1387 |
) |
| 1388 |
if hasattr(expression, "get_source_expressions"): |
| 1389 |
for expr in expression.get_source_expressions(): |
| 1390 |
self.check_filterable(expr) |
| 1391 |
|
| 1392 |
def build_lookup(self, lookups, lhs, rhs): |
| 1393 |
""" |
| 1394 |
Try to extract transforms and lookup from given lhs. |
| 1395 |
|
| 1396 |
The lhs value is something that works like SQLExpression. |
| 1397 |
The rhs value is what the lookup is going to compare against. |
| 1398 |
The lookups is a list of names to extract using get_lookup() |
| 1399 |
and get_transform(). |
| 1400 |
""" |
| 1401 |
# __exact is the default lookup if one isn't given. |
| 1402 |
*transforms, lookup_name = lookups or ["exact"] |
| 1403 |
for name in transforms: |
| 1404 |
lhs = self.try_transform(lhs, name, lookups) |
| 1405 |
# First try get_lookup() so that the lookup takes precedence if the lhs |
| 1406 |
# supports both transform and lookup for the name. |
| 1407 |
lookup_class = lhs.get_lookup(lookup_name) |
| 1408 |
if not lookup_class: |
| 1409 |
# A lookup wasn't found. Try to interpret the name as a transform |
| 1410 |
# and do an Exact lookup against it. |
| 1411 |
lhs = self.try_transform(lhs, lookup_name) |
| 1412 |
lookup_name = "exact" |
| 1413 |
lookup_class = lhs.get_lookup(lookup_name) |
| 1414 |
if not lookup_class: |
| 1415 |
return |
| 1416 |
|
| 1417 |
lookup = lookup_class(lhs, rhs) |
| 1418 |
# Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all |
| 1419 |
# uses of None as a query value unless the lookup supports it. |
| 1420 |
if lookup.rhs is None and not lookup.can_use_none_as_rhs: |
| 1421 |
if lookup_name not in ("exact", "iexact"): |
| 1422 |
raise ValueError("Cannot use None as a query value") |
| 1423 |
return lhs.get_lookup("isnull")(lhs, True) |
| 1424 |
|
| 1425 |
# For Oracle '' is equivalent to null. The check must be done at this |
| 1426 |
# stage because join promotion can't be done in the compiler. Using |
| 1427 |
# DEFAULT_DB_ALIAS isn't nice but it's the best that can be done here. |
| 1428 |
# A similar thing is done in is_nullable(), too. |
| 1429 |
if ( |
| 1430 |
lookup_name == "exact" |
| 1431 |
and lookup.rhs == "" |
| 1432 |
and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls |
| 1433 |
): |
| 1434 |
return lhs.get_lookup("isnull")(lhs, True) |
| 1435 |
|
| 1436 |
return lookup |
| 1437 |
|
| 1438 |
def try_transform(self, lhs, name, lookups=None): |
| 1439 |
""" |
| 1440 |
Helper method for build_lookup(). Try to fetch and initialize |
| 1441 |
a transform for name parameter from lhs. |
| 1442 |
""" |
| 1443 |
transform_class = lhs.get_transform(name) |
| 1444 |
if transform_class: |
| 1445 |
return transform_class(lhs) |
| 1446 |
else: |
| 1447 |
output_field = lhs.output_field.__class__ |
| 1448 |
suggested_lookups = difflib.get_close_matches( |
| 1449 |
name, lhs.output_field.get_lookups() |
| 1450 |
) |
| 1451 |
if suggested_lookups: |
| 1452 |
suggestion = ", perhaps you meant %s?" % " or ".join(suggested_lookups) |
| 1453 |
else: |
| 1454 |
suggestion = "." |
| 1455 |
if lookups is not None: |
| 1456 |
name_index = lookups.index(name) |
| 1457 |
unsupported_lookup = LOOKUP_SEP.join(lookups[name_index:]) |
| 1458 |
else: |
| 1459 |
unsupported_lookup = name |
| 1460 |
raise FieldError( |
| 1461 |
"Unsupported lookup '%s' for %s or join on the field not " |
| 1462 |
"permitted%s" % (unsupported_lookup, output_field.__name__, suggestion) |
| 1463 |
) |
| 1464 |
|
| 1465 |
def build_filter( |
| 1466 |
self, |
| 1467 |
filter_expr, |
| 1468 |
branch_negated=False, |
| 1469 |
current_negated=False, |
| 1470 |
can_reuse=None, |
| 1471 |
allow_joins=True, |
| 1472 |
split_subq=True, |
| 1473 |
check_filterable=True, |
| 1474 |
summarize=False, |
| 1475 |
update_join_types=True, |
| 1476 |
): |
| 1477 |
""" |
| 1478 |
Build a WhereNode for a single filter clause but don't add it |
| 1479 |
to this Query. Query.add_q() will then add this filter to the where |
| 1480 |
Node. |
| 1481 |
|
| 1482 |
The 'branch_negated' tells us if the current branch contains any |
| 1483 |
negations. This will be used to determine if subqueries are needed. |
| 1484 |
|
| 1485 |
The 'current_negated' is used to determine if the current filter is |
| 1486 |
negated or not and this will be used to determine if IS NULL filtering |
| 1487 |
is needed. |
| 1488 |
|
| 1489 |
The difference between current_negated and branch_negated is that |
| 1490 |
branch_negated is set on first negation, but current_negated is |
| 1491 |
flipped for each negation. |
| 1492 |
|
| 1493 |
Note that add_filter will not do any negating itself, that is done |
| 1494 |
upper in the code by add_q(). |
| 1495 |
|
| 1496 |
The 'can_reuse' is a set of reusable joins for multijoins. |
| 1497 |
|
| 1498 |
The method will create a filter clause that can be added to the current |
| 1499 |
query. However, if the filter isn't added to the query then the caller |
| 1500 |
is responsible for unreffing the joins used. |
| 1501 |
""" |
| 1502 |
if isinstance(filter_expr, dict): |
| 1503 |
raise FieldError("Cannot parse keyword query as dict") |
| 1504 |
if isinstance(filter_expr, Q): |
| 1505 |
return self._add_q( |
| 1506 |
filter_expr, |
| 1507 |
branch_negated=branch_negated, |
| 1508 |
current_negated=current_negated, |
| 1509 |
used_aliases=can_reuse, |
| 1510 |
allow_joins=allow_joins, |
| 1511 |
split_subq=split_subq, |
| 1512 |
check_filterable=check_filterable, |
| 1513 |
summarize=summarize, |
| 1514 |
update_join_types=update_join_types, |
| 1515 |
) |
| 1516 |
if hasattr(filter_expr, "resolve_expression"): |
| 1517 |
if not getattr(filter_expr, "conditional", False): |
| 1518 |
raise TypeError("Cannot filter against a non-conditional expression.") |
| 1519 |
condition = filter_expr.resolve_expression( |
| 1520 |
self, allow_joins=allow_joins, reuse=can_reuse, summarize=summarize |
| 1521 |
) |
| 1522 |
if not isinstance(condition, Lookup): |
| 1523 |
condition = self.build_lookup(["exact"], condition, True) |
| 1524 |
return WhereNode([condition], connector=AND), [] |
| 1525 |
arg, value = filter_expr |
| 1526 |
if not arg: |
| 1527 |
raise FieldError("Cannot parse keyword query %r" % arg) |
| 1528 |
lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize) |
| 1529 |
|
| 1530 |
if check_filterable: |
| 1531 |
self.check_filterable(reffed_expression) |
| 1532 |
|
| 1533 |
if not allow_joins and len(parts) > 1: |
| 1534 |
raise FieldError("Joined field references are not permitted in this query") |
| 1535 |
|
| 1536 |
pre_joins = self.alias_refcount.copy() |
| 1537 |
value = self.resolve_lookup_value(value, can_reuse, allow_joins, summarize) |
| 1538 |
used_joins = { |
| 1539 |
k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0) |
| 1540 |
} |
| 1541 |
|
| 1542 |
if check_filterable: |
| 1543 |
self.check_filterable(value) |
| 1544 |
|
| 1545 |
if reffed_expression: |
| 1546 |
condition = self.build_lookup(lookups, reffed_expression, value) |
| 1547 |
return WhereNode([condition], connector=AND), [] |
| 1548 |
|
| 1549 |
opts = self.get_meta() |
| 1550 |
alias = self.get_initial_alias() |
| 1551 |
allow_many = not branch_negated or not split_subq |
| 1552 |
|
| 1553 |
try: |
| 1554 |
join_info = self.setup_joins( |
| 1555 |
parts, |
| 1556 |
opts, |
| 1557 |
alias, |
| 1558 |
can_reuse=can_reuse, |
| 1559 |
allow_many=allow_many, |
| 1560 |
) |
| 1561 |
|
| 1562 |
# Prevent iterator from being consumed by check_related_objects() |
| 1563 |
if isinstance(value, Iterator): |
| 1564 |
value = list(value) |
| 1565 |
self.check_related_objects(join_info.final_field, value, join_info.opts) |
| 1566 |
|
| 1567 |
# split_exclude() needs to know which joins were generated for the |
| 1568 |
# lookup parts |
| 1569 |
self._lookup_joins = join_info.joins |
| 1570 |
except MultiJoin as e: |
| 1571 |
return self.split_exclude(filter_expr, can_reuse, e.names_with_path) |
| 1572 |
|
| 1573 |
# Update used_joins before trimming since they are reused to determine |
| 1574 |
# which joins could be later promoted to INNER. |
| 1575 |
used_joins.update(join_info.joins) |
| 1576 |
targets, alias, join_list = self.trim_joins( |
| 1577 |
join_info.targets, join_info.joins, join_info.path |
| 1578 |
) |
| 1579 |
if can_reuse is not None: |
| 1580 |
can_reuse.update(join_list) |
| 1581 |
|
| 1582 |
if join_info.final_field.is_relation: |
| 1583 |
if len(targets) == 1: |
| 1584 |
col = self._get_col(targets[0], join_info.final_field, alias) |
| 1585 |
else: |
| 1586 |
col = ColPairs(alias, targets, join_info.targets, join_info.final_field) |
| 1587 |
else: |
| 1588 |
col = self._get_col(targets[0], join_info.final_field, alias) |
| 1589 |
|
| 1590 |
condition = self.build_lookup(lookups, col, value) |
| 1591 |
lookup_type = condition.lookup_name |
| 1592 |
clause = WhereNode([condition], connector=AND) |
| 1593 |
|
| 1594 |
require_outer = ( |
| 1595 |
lookup_type == "isnull" and condition.rhs is True and not current_negated |
| 1596 |
) |
| 1597 |
if ( |
| 1598 |
current_negated |
| 1599 |
and (lookup_type != "isnull" or condition.rhs is False) |
| 1600 |
and condition.rhs is not None |
| 1601 |
): |
| 1602 |
require_outer = True |
| 1603 |
if lookup_type != "isnull": |
| 1604 |
# The condition added here will be SQL like this: |
| 1605 |
# NOT (col IS NOT NULL), where the first NOT is added in |
| 1606 |
# upper layers of code. The reason for addition is that if col |
| 1607 |
# is null, then col != someval will result in SQL "unknown" |
| 1608 |
# which isn't the same as in Python. The Python None handling |
| 1609 |
# is wanted, and it can be gotten by |
| 1610 |
# (col IS NULL OR col != someval) |
| 1611 |
# <=> |
| 1612 |
# NOT (col IS NOT NULL AND col = someval). |
| 1613 |
if ( |
| 1614 |
self.is_nullable(targets[0]) |
| 1615 |
or self.alias_map[join_list[-1]].join_type == LOUTER |
| 1616 |
): |
| 1617 |
lookup_class = targets[0].get_lookup("isnull") |
| 1618 |
col = self._get_col(targets[0], join_info.targets[0], alias) |
| 1619 |
clause.add(lookup_class(col, False), AND) |
| 1620 |
# If someval is a nullable column, someval IS NOT NULL is |
| 1621 |
# added. |
| 1622 |
if isinstance(value, Col) and self.is_nullable(value.target): |
| 1623 |
lookup_class = value.target.get_lookup("isnull") |
| 1624 |
clause.add(lookup_class(value, False), AND) |
| 1625 |
return clause, used_joins if not require_outer else () |
| 1626 |
|
| 1627 |
def add_filter(self, filter_lhs, filter_rhs): |
| 1628 |
self.add_q(Q((filter_lhs, filter_rhs))) |
| 1629 |
|
| 1630 |
def add_q(self, q_object, reuse_all=False): |
| 1631 |
""" |
| 1632 |
A preprocessor for the internal _add_q(). Responsible for doing final |
| 1633 |
join promotion. |
| 1634 |
""" |
| 1635 |
# For join promotion this case is doing an AND for the added q_object |
| 1636 |
# and existing conditions. So, any existing inner join forces the join |
| 1637 |
# type to remain inner. Existing outer joins can however be demoted. |
| 1638 |
# (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if |
| 1639 |
# rel_a doesn't produce any rows, then the whole condition must fail. |
| 1640 |
# So, demotion is OK. |
| 1641 |
existing_inner = { |
| 1642 |
a for a in self.alias_map if self.alias_map[a].join_type == INNER |
| 1643 |
} |
| 1644 |
if reuse_all: |
| 1645 |
can_reuse = set(self.alias_map) |
| 1646 |
else: |
| 1647 |
can_reuse = self.used_aliases |
| 1648 |
clause, _ = self._add_q(q_object, can_reuse) |
| 1649 |
if clause: |
| 1650 |
self.where.add(clause, AND) |
| 1651 |
self.demote_joins(existing_inner) |
| 1652 |
|
| 1653 |
def build_where(self, filter_expr): |
| 1654 |
return self.build_filter(filter_expr, allow_joins=False)[0] |
| 1655 |
|
| 1656 |
def clear_where(self): |
| 1657 |
self.where = WhereNode() |
| 1658 |
|
| 1659 |
def _add_q( |
| 1660 |
self, |
| 1661 |
q_object, |
| 1662 |
used_aliases, |
| 1663 |
branch_negated=False, |
| 1664 |
current_negated=False, |
| 1665 |
allow_joins=True, |
| 1666 |
split_subq=True, |
| 1667 |
check_filterable=True, |
| 1668 |
summarize=False, |
| 1669 |
update_join_types=True, |
| 1670 |
): |
| 1671 |
"""Add a Q-object to the current filter.""" |
| 1672 |
connector = q_object.connector |
| 1673 |
current_negated ^= q_object.negated |
| 1674 |
branch_negated = branch_negated or q_object.negated |
| 1675 |
target_clause = WhereNode(connector=connector, negated=q_object.negated) |
| 1676 |
joinpromoter = JoinPromoter( |
| 1677 |
q_object.connector, len(q_object.children), current_negated |
| 1678 |
) |
| 1679 |
for child in q_object.children: |
| 1680 |
child_clause, needed_inner = self.build_filter( |
| 1681 |
child, |
| 1682 |
can_reuse=used_aliases, |
| 1683 |
branch_negated=branch_negated, |
| 1684 |
current_negated=current_negated, |
| 1685 |
allow_joins=allow_joins, |
| 1686 |
split_subq=split_subq, |
| 1687 |
check_filterable=check_filterable, |
| 1688 |
summarize=summarize, |
| 1689 |
update_join_types=update_join_types, |
| 1690 |
) |
| 1691 |
joinpromoter.add_votes(needed_inner) |
| 1692 |
if child_clause: |
| 1693 |
target_clause.add(child_clause, connector) |
| 1694 |
if update_join_types: |
| 1695 |
needed_inner = joinpromoter.update_join_types(self) |
| 1696 |
else: |
| 1697 |
needed_inner = [] |
| 1698 |
return target_clause, needed_inner |
| 1699 |
|
| 1700 |
def add_filtered_relation(self, filtered_relation, alias): |
| 1701 |
if "." in alias: |
| 1702 |
raise ValueError( |
| 1703 |
"FilteredRelation doesn't support aliases with periods " |
| 1704 |
"(got %r)." % alias |
| 1705 |
) |
| 1706 |
self.check_alias(alias) |
| 1707 |
filtered_relation.alias = alias |
| 1708 |
relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type( |
| 1709 |
filtered_relation.relation_name |
| 1710 |
) |
| 1711 |
if relation_lookup_parts: |
| 1712 |
raise ValueError( |
| 1713 |
"FilteredRelation's relation_name cannot contain lookups " |
| 1714 |
"(got %r)." % filtered_relation.relation_name |
| 1715 |
) |
| 1716 |
for lookup in get_children_from_q(filtered_relation.condition): |
| 1717 |
lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup) |
| 1718 |
shift = 2 if not lookup_parts else 1 |
| 1719 |
lookup_field_path = lookup_field_parts[:-shift] |
| 1720 |
for idx, lookup_field_part in enumerate(lookup_field_path): |
| 1721 |
if len(relation_field_parts) > idx: |
| 1722 |
if relation_field_parts[idx] != lookup_field_part: |
| 1723 |
raise ValueError( |
| 1724 |
"FilteredRelation's condition doesn't support " |
| 1725 |
"relations outside the %r (got %r)." |
| 1726 |
% (filtered_relation.relation_name, lookup) |
| 1727 |
) |
| 1728 |
if len(lookup_field_parts) > len(relation_field_parts) + 1: |
| 1729 |
raise ValueError( |
| 1730 |
"FilteredRelation's condition doesn't support nested " |
| 1731 |
"relations deeper than the relation_name (got %r for " |
| 1732 |
"%r)." % (lookup, filtered_relation.relation_name) |
| 1733 |
) |
| 1734 |
filtered_relation.condition = rename_prefix_from_q( |
| 1735 |
filtered_relation.relation_name, |
| 1736 |
alias, |
| 1737 |
filtered_relation.condition, |
| 1738 |
) |
| 1739 |
self._filtered_relations[filtered_relation.alias] = filtered_relation |
| 1740 |
|
| 1741 |
def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False): |
| 1742 |
""" |
| 1743 |
Walk the list of names and turns them into PathInfo tuples. A single |
| 1744 |
name in 'names' can generate multiple PathInfos (m2m, for example). |
| 1745 |
|
| 1746 |
'names' is the path of names to travel, 'opts' is the model Options we |
| 1747 |
start the name resolving from, 'allow_many' is as for setup_joins(). |
| 1748 |
If fail_on_missing is set to True, then a name that can't be resolved |
| 1749 |
will generate a FieldError. |
| 1750 |
|
| 1751 |
Return a list of PathInfo tuples. In addition return the final field |
| 1752 |
(the last used join field) and target (which is a field guaranteed to |
| 1753 |
contain the same value as the final field). Finally, return those names |
| 1754 |
that weren't found (which are likely transforms and the final lookup). |
| 1755 |
""" |
| 1756 |
path, names_with_path = [], [] |
| 1757 |
for pos, name in enumerate(names): |
| 1758 |
cur_names_with_path = (name, []) |
| 1759 |
if name == "pk" and opts is not None: |
| 1760 |
name = opts.pk.name |
| 1761 |
|
| 1762 |
field = None |
| 1763 |
filtered_relation = None |
| 1764 |
try: |
| 1765 |
if opts is None: |
| 1766 |
raise FieldDoesNotExist |
| 1767 |
field = opts.get_field(name) |
| 1768 |
except FieldDoesNotExist: |
| 1769 |
if name in self.annotation_select: |
| 1770 |
field = self.annotation_select[name].output_field |
| 1771 |
elif name in self._filtered_relations and pos == 0: |
| 1772 |
filtered_relation = self._filtered_relations[name] |
| 1773 |
if LOOKUP_SEP in filtered_relation.relation_name: |
| 1774 |
parts = filtered_relation.relation_name.split(LOOKUP_SEP) |
| 1775 |
filtered_relation_path, field, _, _ = self.names_to_path( |
| 1776 |
parts, |
| 1777 |
opts, |
| 1778 |
allow_many, |
| 1779 |
fail_on_missing, |
| 1780 |
) |
| 1781 |
path.extend(filtered_relation_path[:-1]) |
| 1782 |
else: |
| 1783 |
field = opts.get_field(filtered_relation.relation_name) |
| 1784 |
if field is not None: |
| 1785 |
# Fields that contain one-to-many relations with a generic |
| 1786 |
# model (like a GenericForeignKey) cannot generate reverse |
| 1787 |
# relations and therefore cannot be used for reverse querying. |
| 1788 |
if field.is_relation and not field.related_model: |
| 1789 |
raise FieldError( |
| 1790 |
"Field %r does not generate an automatic reverse " |
| 1791 |
"relation and therefore cannot be used for reverse " |
| 1792 |
"querying. If it is a GenericForeignKey, consider " |
| 1793 |
"adding a GenericRelation." % name |
| 1794 |
) |
| 1795 |
try: |
| 1796 |
model = field.model._meta.concrete_model |
| 1797 |
except AttributeError: |
| 1798 |
# QuerySet.annotate() may introduce fields that aren't |
| 1799 |
# attached to a model. |
| 1800 |
model = None |
| 1801 |
else: |
| 1802 |
# We didn't find the current field, so move position back |
| 1803 |
# one step. |
| 1804 |
pos -= 1 |
| 1805 |
if pos == -1 or fail_on_missing: |
| 1806 |
available = sorted( |
| 1807 |
[ |
| 1808 |
*get_field_names_from_opts(opts), |
| 1809 |
*self.annotation_select, |
| 1810 |
*self._filtered_relations, |
| 1811 |
] |
| 1812 |
) |
| 1813 |
raise FieldError( |
| 1814 |
"Cannot resolve keyword '%s' into field. " |
| 1815 |
"Choices are: %s" % (name, ", ".join(available)) |
| 1816 |
) |
| 1817 |
break |
| 1818 |
# Check if we need any joins for concrete inheritance cases (the |
| 1819 |
# field lives in parent, but we are currently in one of its |
| 1820 |
# children) |
| 1821 |
if opts is not None and model is not opts.model: |
| 1822 |
path_to_parent = opts.get_path_to_parent(model) |
| 1823 |
if path_to_parent: |
| 1824 |
path.extend(path_to_parent) |
| 1825 |
cur_names_with_path[1].extend(path_to_parent) |
| 1826 |
opts = path_to_parent[-1].to_opts |
| 1827 |
if hasattr(field, "path_infos"): |
| 1828 |
if filtered_relation: |
| 1829 |
pathinfos = field.get_path_info(filtered_relation) |
| 1830 |
else: |
| 1831 |
pathinfos = field.path_infos |
| 1832 |
if not allow_many: |
| 1833 |
for inner_pos, p in enumerate(pathinfos): |
| 1834 |
if p.m2m: |
| 1835 |
cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1]) |
| 1836 |
names_with_path.append(cur_names_with_path) |
| 1837 |
raise MultiJoin(pos + 1, names_with_path) |
| 1838 |
last = pathinfos[-1] |
| 1839 |
path.extend(pathinfos) |
| 1840 |
final_field = last.join_field |
| 1841 |
opts = last.to_opts |
| 1842 |
targets = last.target_fields |
| 1843 |
cur_names_with_path[1].extend(pathinfos) |
| 1844 |
names_with_path.append(cur_names_with_path) |
| 1845 |
else: |
| 1846 |
# Local non-relational field. |
| 1847 |
final_field = field |
| 1848 |
targets = (field,) |
| 1849 |
if fail_on_missing and pos + 1 != len(names): |
| 1850 |
raise FieldError( |
| 1851 |
"Cannot resolve keyword %r into field. Join on '%s'" |
| 1852 |
" not permitted." % (names[pos + 1], name) |
| 1853 |
) |
| 1854 |
break |
| 1855 |
return path, final_field, targets, names[pos + 1 :] |
| 1856 |
|
| 1857 |
def setup_joins( |
| 1858 |
self, |
| 1859 |
names, |
| 1860 |
opts, |
| 1861 |
alias, |
| 1862 |
can_reuse=None, |
| 1863 |
allow_many=True, |
| 1864 |
): |
| 1865 |
""" |
| 1866 |
Compute the necessary table joins for the passage through the fields |
| 1867 |
given in 'names'. 'opts' is the Options class for the current model |
| 1868 |
(which gives the table we are starting from), 'alias' is the alias for |
| 1869 |
the table to start the joining from. |
| 1870 |
|
| 1871 |
The 'can_reuse' defines the reverse foreign key joins we can reuse. It |
| 1872 |
can be None in which case all joins are reusable or a set of aliases |
| 1873 |
that can be reused. Note that non-reverse foreign keys are always |
| 1874 |
reusable when using setup_joins(). |
| 1875 |
|
| 1876 |
If 'allow_many' is False, then any reverse foreign key seen will |
| 1877 |
generate a MultiJoin exception. |
| 1878 |
|
| 1879 |
Return the final field involved in the joins, the target field (used |
| 1880 |
for any 'where' constraint), the final 'opts' value, the joins, the |
| 1881 |
field path traveled to generate the joins, and a transform function |
| 1882 |
that takes a field and alias and is equivalent to `field.get_col(alias)` |
| 1883 |
in the simple case but wraps field transforms if they were included in |
| 1884 |
names. |
| 1885 |
|
| 1886 |
The target field is the field containing the concrete value. Final |
| 1887 |
field can be something different, for example foreign key pointing to |
| 1888 |
that value. Final field is needed for example in some value |
| 1889 |
conversions (convert 'obj' in fk__id=obj to pk val using the foreign |
| 1890 |
key field for example). |
| 1891 |
""" |
| 1892 |
joins = [alias] |
| 1893 |
# The transform can't be applied yet, as joins must be trimmed later. |
| 1894 |
# To avoid making every caller of this method look up transforms |
| 1895 |
# directly, compute transforms here and create a partial that converts |
| 1896 |
# fields to the appropriate wrapped version. |
| 1897 |
|
| 1898 |
def final_transformer(field, alias): |
| 1899 |
if not self.alias_cols: |
| 1900 |
alias = None |
| 1901 |
return field.get_col(alias) |
| 1902 |
|
| 1903 |
# Try resolving all the names as fields first. If there's an error, |
| 1904 |
# treat trailing names as lookups until a field can be resolved. |
| 1905 |
last_field_exception = None |
| 1906 |
for pivot in range(len(names), 0, -1): |
| 1907 |
try: |
| 1908 |
path, final_field, targets, rest = self.names_to_path( |
| 1909 |
names[:pivot], |
| 1910 |
opts, |
| 1911 |
allow_many, |
| 1912 |
fail_on_missing=True, |
| 1913 |
) |
| 1914 |
except FieldError as exc: |
| 1915 |
if pivot == 1: |
| 1916 |
# The first item cannot be a lookup, so it's safe |
| 1917 |
# to raise the field error here. |
| 1918 |
raise |
| 1919 |
else: |
| 1920 |
last_field_exception = exc |
| 1921 |
else: |
| 1922 |
# The transforms are the remaining items that couldn't be |
| 1923 |
# resolved into fields. |
| 1924 |
transforms = names[pivot:] |
| 1925 |
break |
| 1926 |
for name in transforms: |
| 1927 |
|
| 1928 |
def transform(field, alias, *, name, previous): |
| 1929 |
try: |
| 1930 |
wrapped = previous(field, alias) |
| 1931 |
return self.try_transform(wrapped, name) |
| 1932 |
except FieldError: |
| 1933 |
# FieldError is raised if the transform doesn't exist. |
| 1934 |
if isinstance(final_field, Field) and last_field_exception: |
| 1935 |
raise last_field_exception |
| 1936 |
else: |
| 1937 |
raise |
| 1938 |
|
| 1939 |
final_transformer = functools.partial( |
| 1940 |
transform, name=name, previous=final_transformer |
| 1941 |
) |
| 1942 |
final_transformer.has_transforms = True |
| 1943 |
# Then, add the path to the query's joins. Note that we can't trim |
| 1944 |
# joins at this stage - we will need the information about join type |
| 1945 |
# of the trimmed joins. |
| 1946 |
for join in path: |
| 1947 |
if join.filtered_relation: |
| 1948 |
filtered_relation = join.filtered_relation.clone() |
| 1949 |
table_alias = filtered_relation.alias |
| 1950 |
else: |
| 1951 |
filtered_relation = None |
| 1952 |
table_alias = None |
| 1953 |
opts = join.to_opts |
| 1954 |
if join.direct: |
| 1955 |
nullable = self.is_nullable(join.join_field) |
| 1956 |
else: |
| 1957 |
nullable = True |
| 1958 |
connection = self.join_class( |
| 1959 |
opts.db_table, |
| 1960 |
alias, |
| 1961 |
table_alias, |
| 1962 |
INNER, |
| 1963 |
join.join_field, |
| 1964 |
nullable, |
| 1965 |
filtered_relation=filtered_relation, |
| 1966 |
) |
| 1967 |
reuse = can_reuse if join.m2m else None |
| 1968 |
alias = self.join(connection, reuse=reuse) |
| 1969 |
joins.append(alias) |
| 1970 |
if join.filtered_relation and can_reuse is not None: |
| 1971 |
can_reuse.add(alias) |
| 1972 |
return JoinInfo(final_field, targets, opts, joins, path, final_transformer) |
| 1973 |
|
| 1974 |
def trim_joins(self, targets, joins, path): |
| 1975 |
""" |
| 1976 |
The 'target' parameter is the final field being joined to, 'joins' |
| 1977 |
is the full list of join aliases. The 'path' contain the PathInfos |
| 1978 |
used to create the joins. |
| 1979 |
|
| 1980 |
Return the final target field and table alias and the new active |
| 1981 |
joins. |
| 1982 |
|
| 1983 |
Always trim any direct join if the target column is already in the |
| 1984 |
previous table. Can't trim reverse joins as it's unknown if there's |
| 1985 |
anything on the other side of the join. |
| 1986 |
""" |
| 1987 |
joins = joins[:] |
| 1988 |
for pos, info in enumerate(reversed(path)): |
| 1989 |
if len(joins) == 1 or not info.direct: |
| 1990 |
break |
| 1991 |
if info.filtered_relation: |
| 1992 |
break |
| 1993 |
join_targets = {t.column for t in info.join_field.foreign_related_fields} |
| 1994 |
cur_targets = {t.column for t in targets} |
| 1995 |
if not cur_targets.issubset(join_targets): |
| 1996 |
break |
| 1997 |
targets_dict = { |
| 1998 |
r[1].column: r[0] |
| 1999 |
for r in info.join_field.related_fields |
| 2000 |
if r[1].column in cur_targets |
| 2001 |
} |
| 2002 |
targets = tuple(targets_dict[t.column] for t in targets) |
| 2003 |
self.unref_alias(joins.pop()) |
| 2004 |
return targets, joins[-1], joins |
| 2005 |
|
| 2006 |
@classmethod |
| 2007 |
def _gen_cols(cls, exprs, include_external=False, resolve_refs=True): |
| 2008 |
for expr in exprs: |
| 2009 |
if isinstance(expr, Col): |
| 2010 |
yield expr |
| 2011 |
elif include_external and callable( |
| 2012 |
getattr(expr, "get_external_cols", None) |
| 2013 |
): |
| 2014 |
yield from expr.get_external_cols() |
| 2015 |
elif hasattr(expr, "get_source_expressions"): |
| 2016 |
if not resolve_refs and isinstance(expr, Ref): |
| 2017 |
continue |
| 2018 |
yield from cls._gen_cols( |
| 2019 |
expr.get_source_expressions(), |
| 2020 |
include_external=include_external, |
| 2021 |
resolve_refs=resolve_refs, |
| 2022 |
) |
| 2023 |
|
| 2024 |
@classmethod |
| 2025 |
def _gen_col_aliases(cls, exprs): |
| 2026 |
yield from (expr.alias for expr in cls._gen_cols(exprs)) |
| 2027 |
|
| 2028 |
def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False): |
| 2029 |
annotation = self.annotations.get(name) |
| 2030 |
if annotation is not None: |
| 2031 |
if not allow_joins: |
| 2032 |
for alias in self._gen_col_aliases([annotation]): |
| 2033 |
if isinstance(self.alias_map[alias], Join): |
| 2034 |
raise FieldError( |
| 2035 |
"Joined field references are not permitted in this query" |
| 2036 |
) |
| 2037 |
if summarize: |
| 2038 |
# Summarize currently means we are doing an aggregate() query |
| 2039 |
# which is executed as a wrapped subquery if any of the |
| 2040 |
# aggregate() elements reference an existing annotation. In |
| 2041 |
# that case we need to return a Ref to the subquery's annotation. |
| 2042 |
if name not in self.annotation_select: |
| 2043 |
raise FieldError( |
| 2044 |
"Cannot aggregate over the '%s' alias. Use annotate() " |
| 2045 |
"to promote it." % name |
| 2046 |
) |
| 2047 |
return Ref(name, self.annotation_select[name]) |
| 2048 |
else: |
| 2049 |
return annotation |
| 2050 |
else: |
| 2051 |
field_list = name.split(LOOKUP_SEP) |
| 2052 |
annotation = self.annotations.get(field_list[0]) |
| 2053 |
if annotation is not None: |
| 2054 |
for transform in field_list[1:]: |
| 2055 |
annotation = self.try_transform(annotation, transform) |
| 2056 |
return annotation |
| 2057 |
join_info = self.setup_joins( |
| 2058 |
field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse |
| 2059 |
) |
| 2060 |
targets, final_alias, join_list = self.trim_joins( |
| 2061 |
join_info.targets, join_info.joins, join_info.path |
| 2062 |
) |
| 2063 |
if not allow_joins and len(join_list) > 1: |
| 2064 |
raise FieldError( |
| 2065 |
"Joined field references are not permitted in this query" |
| 2066 |
) |
| 2067 |
if len(targets) > 1: |
| 2068 |
raise FieldError( |
| 2069 |
"Referencing multicolumn fields with F() objects isn't supported" |
| 2070 |
) |
| 2071 |
# Verify that the last lookup in name is a field or a transform: |
| 2072 |
# transform_function() raises FieldError if not. |
| 2073 |
transform = join_info.transform_function(targets[0], final_alias) |
| 2074 |
if reuse is not None: |
| 2075 |
reuse.update(join_list) |
| 2076 |
return transform |
| 2077 |
|
| 2078 |
def split_exclude(self, filter_expr, can_reuse, names_with_path): |
| 2079 |
""" |
| 2080 |
When doing an exclude against any kind of N-to-many relation, we need |
| 2081 |
to use a subquery. This method constructs the nested query, given the |
| 2082 |
original exclude filter (filter_expr) and the portion up to the first |
| 2083 |
N-to-many relation field. |
| 2084 |
|
| 2085 |
For example, if the origin filter is ~Q(child__name='foo'), filter_expr |
| 2086 |
is ('child__name', 'foo') and can_reuse is a set of joins usable for |
| 2087 |
filters in the original query. |
| 2088 |
|
| 2089 |
We will turn this into equivalent of: |
| 2090 |
WHERE NOT EXISTS( |
| 2091 |
SELECT 1 |
| 2092 |
FROM child |
| 2093 |
WHERE name = 'foo' AND child.parent_id = parent.id |
| 2094 |
LIMIT 1 |
| 2095 |
) |
| 2096 |
""" |
| 2097 |
# Generate the inner query. |
| 2098 |
query = self.__class__(self.model) |
| 2099 |
query._filtered_relations = self._filtered_relations |
| 2100 |
filter_lhs, filter_rhs = filter_expr |
| 2101 |
if isinstance(filter_rhs, OuterRef): |
| 2102 |
filter_rhs = OuterRef(filter_rhs) |
| 2103 |
elif isinstance(filter_rhs, F): |
| 2104 |
filter_rhs = OuterRef(filter_rhs.name) |
| 2105 |
query.add_filter(filter_lhs, filter_rhs) |
| 2106 |
query.clear_ordering(force=True) |
| 2107 |
# Try to have as simple as possible subquery -> trim leading joins from |
| 2108 |
# the subquery. |
| 2109 |
trimmed_prefix, contains_louter = query.trim_start(names_with_path) |
| 2110 |
|
| 2111 |
col = query.select[0] |
| 2112 |
select_field = col.target |
| 2113 |
alias = col.alias |
| 2114 |
if alias in can_reuse: |
| 2115 |
pk = select_field.model._meta.pk |
| 2116 |
# Need to add a restriction so that outer query's filters are in effect for |
| 2117 |
# the subquery, too. |
| 2118 |
query.bump_prefix(self) |
| 2119 |
lookup_class = select_field.get_lookup("exact") |
| 2120 |
# Note that the query.select[0].alias is different from alias |
| 2121 |
# due to bump_prefix above. |
| 2122 |
lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias)) |
| 2123 |
query.where.add(lookup, AND) |
| 2124 |
query.external_aliases[alias] = True |
| 2125 |
else: |
| 2126 |
lookup_class = select_field.get_lookup("exact") |
| 2127 |
lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix)) |
| 2128 |
query.where.add(lookup, AND) |
| 2129 |
|
| 2130 |
condition, needed_inner = self.build_filter(Exists(query)) |
| 2131 |
|
| 2132 |
if contains_louter: |
| 2133 |
or_null_condition, _ = self.build_filter( |
| 2134 |
("%s__isnull" % trimmed_prefix, True), |
| 2135 |
current_negated=True, |
| 2136 |
branch_negated=True, |
| 2137 |
can_reuse=can_reuse, |
| 2138 |
) |
| 2139 |
condition.add(or_null_condition, OR) |
| 2140 |
# Note that the end result will be: |
| 2141 |
# NOT EXISTS (inner_q) OR outercol IS NULL |
| 2142 |
# this might look crazy but due to how NULL works, this seems to be |
| 2143 |
# correct. If the IS NULL check is removed, then if outercol |
| 2144 |
# IS NULL we will not match the row. |
| 2145 |
return condition, needed_inner |
| 2146 |
|
| 2147 |
def set_empty(self): |
| 2148 |
self.where.add(NothingNode(), AND) |
| 2149 |
for query in self.combined_queries: |
| 2150 |
query.set_empty() |
| 2151 |
|
| 2152 |
def is_empty(self): |
| 2153 |
return any(isinstance(c, NothingNode) for c in self.where.children) |
| 2154 |
|
| 2155 |
def set_limits(self, low=None, high=None): |
| 2156 |
""" |
| 2157 |
Adjust the limits on the rows retrieved. Use low/high to set these, |
| 2158 |
as it makes it more Pythonic to read and write. When the SQL query is |
| 2159 |
created, convert them to the appropriate offset and limit values. |
| 2160 |
|
| 2161 |
Apply any limits passed in here to the existing constraints. Add low |
| 2162 |
to the current low value and clamp both to any existing high value. |
| 2163 |
""" |
| 2164 |
if high is not None: |
| 2165 |
if self.high_mark is not None: |
| 2166 |
self.high_mark = min(self.high_mark, self.low_mark + high) |
| 2167 |
else: |
| 2168 |
self.high_mark = self.low_mark + high |
| 2169 |
if low is not None: |
| 2170 |
if self.high_mark is not None: |
| 2171 |
self.low_mark = min(self.high_mark, self.low_mark + low) |
| 2172 |
else: |
| 2173 |
self.low_mark = self.low_mark + low |
| 2174 |
|
| 2175 |
if self.low_mark == self.high_mark: |
| 2176 |
self.set_empty() |
| 2177 |
|
| 2178 |
def clear_limits(self): |
| 2179 |
"""Clear any existing limits.""" |
| 2180 |
self.low_mark, self.high_mark = 0, None |
| 2181 |
|
| 2182 |
@property |
| 2183 |
def is_sliced(self): |
| 2184 |
return self.low_mark != 0 or self.high_mark is not None |
| 2185 |
|
| 2186 |
def has_limit_one(self): |
| 2187 |
return self.high_mark is not None and (self.high_mark - self.low_mark) == 1 |
| 2188 |
|
| 2189 |
def can_filter(self): |
| 2190 |
""" |
| 2191 |
Return True if adding filters to this instance is still possible. |
| 2192 |
|
| 2193 |
Typically, this means no limits or offsets have been put on the results. |
| 2194 |
""" |
| 2195 |
return not self.is_sliced |
| 2196 |
|
| 2197 |
def clear_select_clause(self): |
| 2198 |
"""Remove all fields from SELECT clause.""" |
| 2199 |
self.select = () |
| 2200 |
self.default_cols = False |
| 2201 |
self.select_related = False |
| 2202 |
self.set_extra_mask(()) |
| 2203 |
self.set_annotation_mask(()) |
| 2204 |
self.selected = None |
| 2205 |
|
| 2206 |
def clear_select_fields(self): |
| 2207 |
""" |
| 2208 |
Clear the list of fields to select (but not extra_select columns). |
| 2209 |
Some queryset types completely replace any existing list of select |
| 2210 |
columns. |
| 2211 |
""" |
| 2212 |
self.select = () |
| 2213 |
self.values_select = () |
| 2214 |
self.selected = None |
| 2215 |
|
| 2216 |
def add_select_col(self, col, name): |
| 2217 |
self.select += (col,) |
| 2218 |
self.values_select += (name,) |
| 2219 |
self.selected[name] = len(self.select) - 1 |
| 2220 |
|
| 2221 |
def set_select(self, cols): |
| 2222 |
self.default_cols = False |
| 2223 |
self.select = tuple(cols) |
| 2224 |
|
| 2225 |
def add_distinct_fields(self, *field_names): |
| 2226 |
""" |
| 2227 |
Add and resolve the given fields to the query's "distinct on" clause. |
| 2228 |
""" |
| 2229 |
self.distinct_fields = field_names |
| 2230 |
self.distinct = True |
| 2231 |
|
| 2232 |
def add_fields(self, field_names, allow_m2m=True): |
| 2233 |
""" |
| 2234 |
Add the given (model) fields to the select set. Add the field names in |
| 2235 |
the order specified. |
| 2236 |
""" |
| 2237 |
alias = self.get_initial_alias() |
| 2238 |
opts = self.get_meta() |
| 2239 |
|
| 2240 |
try: |
| 2241 |
cols = [] |
| 2242 |
for name in field_names: |
| 2243 |
# Join promotion note - we must not remove any rows here, so |
| 2244 |
# if there is no existing joins, use outer join. |
| 2245 |
join_info = self.setup_joins( |
| 2246 |
name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m |
| 2247 |
) |
| 2248 |
targets, final_alias, joins = self.trim_joins( |
| 2249 |
join_info.targets, |
| 2250 |
join_info.joins, |
| 2251 |
join_info.path, |
| 2252 |
) |
| 2253 |
if len(targets) > 1: |
| 2254 |
transformed_targets = [ |
| 2255 |
join_info.transform_function(target, final_alias) |
| 2256 |
for target in targets |
| 2257 |
] |
| 2258 |
cols.append( |
| 2259 |
ColPairs( |
| 2260 |
final_alias if self.alias_cols else None, |
| 2261 |
[col.target for col in transformed_targets], |
| 2262 |
[col.output_field for col in transformed_targets], |
| 2263 |
join_info.final_field, |
| 2264 |
) |
| 2265 |
) |
| 2266 |
else: |
| 2267 |
cols.append(join_info.transform_function(targets[0], final_alias)) |
| 2268 |
if cols: |
| 2269 |
self.set_select(cols) |
| 2270 |
except MultiJoin: |
| 2271 |
raise FieldError("Invalid field name: '%s'" % name) |
| 2272 |
except FieldError: |
| 2273 |
if LOOKUP_SEP in name: |
| 2274 |
# For lookups spanning over relationships, show the error |
| 2275 |
# from the model on which the lookup failed. |
| 2276 |
raise |
| 2277 |
else: |
| 2278 |
names = sorted( |
| 2279 |
[ |
| 2280 |
*get_field_names_from_opts(opts), |
| 2281 |
*self.extra, |
| 2282 |
*self.annotation_select, |
| 2283 |
*self._filtered_relations, |
| 2284 |
] |
| 2285 |
) |
| 2286 |
raise FieldError( |
| 2287 |
"Cannot resolve keyword %r into field. " |
| 2288 |
"Choices are: %s" % (name, ", ".join(names)) |
| 2289 |
) |
| 2290 |
|
| 2291 |
def add_ordering(self, *ordering): |
| 2292 |
""" |
| 2293 |
Add items from the 'ordering' sequence to the query's "order by" |
| 2294 |
clause. These items are either field names (not column names) -- |
| 2295 |
possibly with a direction prefix ('-' or '?') -- or OrderBy |
| 2296 |
expressions. |
| 2297 |
|
| 2298 |
If 'ordering' is empty, clear all ordering from the query. |
| 2299 |
""" |
| 2300 |
errors = [] |
| 2301 |
for item in ordering: |
| 2302 |
if isinstance(item, str): |
| 2303 |
if item == "?": |
| 2304 |
continue |
| 2305 |
item = item.removeprefix("-") |
| 2306 |
if item in self.annotations: |
| 2307 |
continue |
| 2308 |
if self.extra and item in self.extra: |
| 2309 |
continue |
| 2310 |
# names_to_path() validates the lookup. A descriptive |
| 2311 |
# FieldError will be raise if it's not. |
| 2312 |
self.names_to_path(item.split(LOOKUP_SEP), self.model._meta) |
| 2313 |
elif not hasattr(item, "resolve_expression"): |
| 2314 |
errors.append(item) |
| 2315 |
if getattr(item, "contains_aggregate", False): |
| 2316 |
raise FieldError( |
| 2317 |
"Using an aggregate in order_by() without also including " |
| 2318 |
"it in annotate() is not allowed: %s" % item |
| 2319 |
) |
| 2320 |
if errors: |
| 2321 |
raise FieldError("Invalid order_by arguments: %s" % errors) |
| 2322 |
if ordering: |
| 2323 |
self.order_by += ordering |
| 2324 |
else: |
| 2325 |
self.default_ordering = False |
| 2326 |
|
| 2327 |
def clear_ordering(self, force=False, clear_default=True): |
| 2328 |
""" |
| 2329 |
Remove any ordering settings if the current query allows it without |
| 2330 |
side effects, set 'force' to True to clear the ordering regardless. |
| 2331 |
If 'clear_default' is True, there will be no ordering in the resulting |
| 2332 |
query (not even the model's default). |
| 2333 |
""" |
| 2334 |
if not force and ( |
| 2335 |
self.is_sliced or self.distinct_fields or self.select_for_update |
| 2336 |
): |
| 2337 |
return |
| 2338 |
self.order_by = () |
| 2339 |
self.extra_order_by = () |
| 2340 |
if clear_default: |
| 2341 |
self.default_ordering = False |
| 2342 |
|
| 2343 |
def set_group_by(self, allow_aliases=True): |
| 2344 |
""" |
| 2345 |
Expand the GROUP BY clause required by the query. |
| 2346 |
|
| 2347 |
This will usually be the set of all non-aggregate fields in the |
| 2348 |
return data. If the database backend supports grouping by the |
| 2349 |
primary key, and the query would be equivalent, the optimization |
| 2350 |
will be made automatically. |
| 2351 |
""" |
| 2352 |
if allow_aliases and self.values_select: |
| 2353 |
# If grouping by aliases is allowed assign selected value aliases |
| 2354 |
# by moving them to annotations. |
| 2355 |
group_by_annotations = {} |
| 2356 |
values_select = {} |
| 2357 |
for alias, expr in zip(self.values_select, self.select): |
| 2358 |
if isinstance(expr, Col): |
| 2359 |
values_select[alias] = expr |
| 2360 |
else: |
| 2361 |
group_by_annotations[alias] = expr |
| 2362 |
self.annotations = {**group_by_annotations, **self.annotations} |
| 2363 |
self.append_annotation_mask(group_by_annotations) |
| 2364 |
self.select = tuple(values_select.values()) |
| 2365 |
self.values_select = tuple(values_select) |
| 2366 |
if self.selected is not None: |
| 2367 |
for index, value_select in enumerate(values_select): |
| 2368 |
self.selected[value_select] = index |
| 2369 |
group_by = list(self.select) |
| 2370 |
for alias, annotation in self.annotation_select.items(): |
| 2371 |
if not (group_by_cols := annotation.get_group_by_cols()): |
| 2372 |
continue |
| 2373 |
if allow_aliases and not annotation.contains_aggregate: |
| 2374 |
group_by.append(Ref(alias, annotation)) |
| 2375 |
else: |
| 2376 |
group_by.extend(group_by_cols) |
| 2377 |
self.group_by = tuple(group_by) |
| 2378 |
|
| 2379 |
def add_select_related(self, fields): |
| 2380 |
""" |
| 2381 |
Set up the select_related data structure so that we only select |
| 2382 |
certain related models (as opposed to all models, when |
| 2383 |
self.select_related=True). |
| 2384 |
""" |
| 2385 |
if isinstance(self.select_related, bool): |
| 2386 |
field_dict = {} |
| 2387 |
else: |
| 2388 |
field_dict = self.select_related |
| 2389 |
for field in fields: |
| 2390 |
d = field_dict |
| 2391 |
for part in field.split(LOOKUP_SEP): |
| 2392 |
d = d.setdefault(part, {}) |
| 2393 |
self.select_related = field_dict |
| 2394 |
|
| 2395 |
def add_extra(self, select, select_params, where, params, tables, order_by): |
| 2396 |
""" |
| 2397 |
Add data to the various extra_* attributes for user-created additions |
| 2398 |
to the query. |
| 2399 |
""" |
| 2400 |
if select: |
| 2401 |
# We need to pair any placeholder markers in the 'select' |
| 2402 |
# dictionary with their parameters in 'select_params' so that |
| 2403 |
# subsequent updates to the select dictionary also adjust the |
| 2404 |
# parameters appropriately. |
| 2405 |
select_pairs = {} |
| 2406 |
if select_params: |
| 2407 |
param_iter = iter(select_params) |
| 2408 |
else: |
| 2409 |
param_iter = iter([]) |
| 2410 |
for name, entry in select.items(): |
| 2411 |
self.check_alias(name) |
| 2412 |
entry = str(entry) |
| 2413 |
entry_params = [] |
| 2414 |
pos = entry.find("%s") |
| 2415 |
while pos != -1: |
| 2416 |
if pos == 0 or entry[pos - 1] != "%": |
| 2417 |
entry_params.append(next(param_iter)) |
| 2418 |
pos = entry.find("%s", pos + 2) |
| 2419 |
select_pairs[name] = (entry, entry_params) |
| 2420 |
self.extra.update(select_pairs) |
| 2421 |
if where or params: |
| 2422 |
self.where.add(ExtraWhere(where, params), AND) |
| 2423 |
if tables: |
| 2424 |
self.extra_tables += tuple(tables) |
| 2425 |
if order_by: |
| 2426 |
self.extra_order_by = order_by |
| 2427 |
|
| 2428 |
def clear_deferred_loading(self): |
| 2429 |
"""Remove any fields from the deferred loading set.""" |
| 2430 |
self.deferred_loading = (frozenset(), True) |
| 2431 |
|
| 2432 |
def add_deferred_loading(self, field_names): |
| 2433 |
""" |
| 2434 |
Add the given list of model field names to the set of fields to |
| 2435 |
exclude from loading from the database when automatic column selection |
| 2436 |
is done. Add the new field names to any existing field names that |
| 2437 |
are deferred (or removed from any existing field names that are marked |
| 2438 |
as the only ones for immediate loading). |
| 2439 |
""" |
| 2440 |
# Fields on related models are stored in the literal double-underscore |
| 2441 |
# format, so that we can use a set datastructure. We do the foo__bar |
| 2442 |
# splitting and handling when computing the SQL column names (as part of |
| 2443 |
# get_columns()). |
| 2444 |
existing, defer = self.deferred_loading |
| 2445 |
if defer: |
| 2446 |
# Add to existing deferred names. |
| 2447 |
self.deferred_loading = existing.union(field_names), True |
| 2448 |
else: |
| 2449 |
# Remove names from the set of any existing "immediate load" names. |
| 2450 |
if new_existing := existing.difference(field_names): |
| 2451 |
self.deferred_loading = new_existing, False |
| 2452 |
else: |
| 2453 |
self.clear_deferred_loading() |
| 2454 |
if new_only := set(field_names).difference(existing): |
| 2455 |
self.deferred_loading = new_only, True |
| 2456 |
|
| 2457 |
def add_immediate_loading(self, field_names): |
| 2458 |
""" |
| 2459 |
Add the given list of model field names to the set of fields to |
| 2460 |
retrieve when the SQL is executed ("immediate loading" fields). The |
| 2461 |
field names replace any existing immediate loading field names. If |
| 2462 |
there are field names already specified for deferred loading, remove |
| 2463 |
those names from the new field_names before storing the new names |
| 2464 |
for immediate loading. (That is, immediate loading overrides any |
| 2465 |
existing immediate values, but respects existing deferrals.) |
| 2466 |
""" |
| 2467 |
existing, defer = self.deferred_loading |
| 2468 |
field_names = set(field_names) |
| 2469 |
if "pk" in field_names: |
| 2470 |
field_names.remove("pk") |
| 2471 |
field_names.add(self.get_meta().pk.name) |
| 2472 |
|
| 2473 |
if defer: |
| 2474 |
# Remove any existing deferred names from the current set before |
| 2475 |
# setting the new names. |
| 2476 |
self.deferred_loading = field_names.difference(existing), False |
| 2477 |
else: |
| 2478 |
# Replace any existing "immediate load" field names. |
| 2479 |
self.deferred_loading = frozenset(field_names), False |
| 2480 |
|
| 2481 |
def set_annotation_mask(self, names): |
| 2482 |
"""Set the mask of annotations that will be returned by the SELECT.""" |
| 2483 |
if names is None: |
| 2484 |
self.annotation_select_mask = None |
| 2485 |
else: |
| 2486 |
self.annotation_select_mask = set(names) |
| 2487 |
if self.selected: |
| 2488 |
# Prune the masked annotations. |
| 2489 |
self.selected = { |
| 2490 |
key: value |
| 2491 |
for key, value in self.selected.items() |
| 2492 |
if not isinstance(value, str) |
| 2493 |
or value in self.annotation_select_mask |
| 2494 |
} |
| 2495 |
# Append the unmasked annotations. |
| 2496 |
for name in names: |
| 2497 |
self.selected[name] = name |
| 2498 |
self._annotation_select_cache = None |
| 2499 |
|
| 2500 |
def append_annotation_mask(self, names): |
| 2501 |
if self.annotation_select_mask is not None: |
| 2502 |
self.set_annotation_mask(self.annotation_select_mask.union(names)) |
| 2503 |
|
| 2504 |
def set_extra_mask(self, names): |
| 2505 |
""" |
| 2506 |
Set the mask of extra select items that will be returned by SELECT. |
| 2507 |
Don't remove them from the Query since they might be used later. |
| 2508 |
""" |
| 2509 |
if names is None: |
| 2510 |
self.extra_select_mask = None |
| 2511 |
else: |
| 2512 |
self.extra_select_mask = set(names) |
| 2513 |
self._extra_select_cache = None |
| 2514 |
|
| 2515 |
@property |
| 2516 |
def has_select_fields(self): |
| 2517 |
return self.selected is not None |
| 2518 |
|
| 2519 |
def set_values(self, fields): |
| 2520 |
self.select_related = False |
| 2521 |
self.clear_deferred_loading() |
| 2522 |
self.clear_select_fields() |
| 2523 |
|
| 2524 |
selected = {} |
| 2525 |
if fields: |
| 2526 |
for field in fields: |
| 2527 |
self.check_alias(field) |
| 2528 |
field_names = [] |
| 2529 |
extra_names = [] |
| 2530 |
annotation_names = [] |
| 2531 |
if not self.extra and not self.annotations: |
| 2532 |
# Shortcut - if there are no extra or annotations, then |
| 2533 |
# the values() clause must be just field names. |
| 2534 |
field_names = list(fields) |
| 2535 |
selected = dict(zip(fields, range(len(fields)))) |
| 2536 |
else: |
| 2537 |
self.default_cols = False |
| 2538 |
for f in fields: |
| 2539 |
if extra := self.extra_select.get(f): |
| 2540 |
extra_names.append(f) |
| 2541 |
selected[f] = RawSQL(*extra) |
| 2542 |
elif f in self.annotation_select: |
| 2543 |
annotation_names.append(f) |
| 2544 |
selected[f] = f |
| 2545 |
elif f in self.annotations: |
| 2546 |
raise FieldError( |
| 2547 |
f"Cannot select the '{f}' alias. Use annotate() to " |
| 2548 |
"promote it." |
| 2549 |
) |
| 2550 |
else: |
| 2551 |
# Call `names_to_path` to ensure a FieldError including |
| 2552 |
# annotations about to be masked as valid choices if |
| 2553 |
# `f` is not resolvable. |
| 2554 |
if self.annotation_select: |
| 2555 |
self.names_to_path(f.split(LOOKUP_SEP), self.model._meta) |
| 2556 |
selected[f] = len(field_names) |
| 2557 |
field_names.append(f) |
| 2558 |
self.set_extra_mask(extra_names) |
| 2559 |
self.set_annotation_mask(annotation_names) |
| 2560 |
else: |
| 2561 |
field_names = [f.attname for f in self.model._meta.concrete_fields] |
| 2562 |
selected = dict.fromkeys(field_names, None) |
| 2563 |
# Selected annotations must be known before setting the GROUP BY |
| 2564 |
# clause. |
| 2565 |
if self.group_by is True: |
| 2566 |
self.add_fields( |
| 2567 |
(f.attname for f in self.model._meta.concrete_fields), False |
| 2568 |
) |
| 2569 |
# Disable GROUP BY aliases to avoid orphaning references to the |
| 2570 |
# SELECT clause which is about to be cleared. |
| 2571 |
self.set_group_by(allow_aliases=False) |
| 2572 |
self.clear_select_fields() |
| 2573 |
elif self.group_by: |
| 2574 |
# Resolve GROUP BY annotation references if they are not part of |
| 2575 |
# the selected fields anymore. |
| 2576 |
group_by = [] |
| 2577 |
for expr in self.group_by: |
| 2578 |
if isinstance(expr, Ref) and expr.refs not in selected: |
| 2579 |
expr = self.annotations[expr.refs] |
| 2580 |
group_by.append(expr) |
| 2581 |
self.group_by = tuple(group_by) |
| 2582 |
|
| 2583 |
self.values_select = tuple(field_names) |
| 2584 |
self.add_fields(field_names, True) |
| 2585 |
self.selected = selected if fields else None |
| 2586 |
|
| 2587 |
@property |
| 2588 |
def annotation_select(self): |
| 2589 |
""" |
| 2590 |
Return the dictionary of aggregate columns that are not masked and |
| 2591 |
should be used in the SELECT clause. Cache this result for performance. |
| 2592 |
""" |
| 2593 |
if self._annotation_select_cache is not None: |
| 2594 |
return self._annotation_select_cache |
| 2595 |
elif not self.annotations: |
| 2596 |
return {} |
| 2597 |
elif self.annotation_select_mask is not None: |
| 2598 |
self._annotation_select_cache = { |
| 2599 |
k: v |
| 2600 |
for k, v in self.annotations.items() |
| 2601 |
if k in self.annotation_select_mask |
| 2602 |
} |
| 2603 |
return self._annotation_select_cache |
| 2604 |
else: |
| 2605 |
return self.annotations |
| 2606 |
|
| 2607 |
@property |
| 2608 |
def extra_select(self): |
| 2609 |
if self._extra_select_cache is not None: |
| 2610 |
return self._extra_select_cache |
| 2611 |
if not self.extra: |
| 2612 |
return {} |
| 2613 |
elif self.extra_select_mask is not None: |
| 2614 |
self._extra_select_cache = { |
| 2615 |
k: v for k, v in self.extra.items() if k in self.extra_select_mask |
| 2616 |
} |
| 2617 |
return self._extra_select_cache |
| 2618 |
else: |
| 2619 |
return self.extra |
| 2620 |
|
| 2621 |
def trim_start(self, names_with_path): |
| 2622 |
""" |
| 2623 |
Trim joins from the start of the join path. The candidates for trim |
| 2624 |
are the PathInfos in names_with_path structure that are m2m joins. |
| 2625 |
|
| 2626 |
Also set the select column so the start matches the join. |
| 2627 |
|
| 2628 |
This method is meant to be used for generating the subquery joins & |
| 2629 |
cols in split_exclude(). |
| 2630 |
|
| 2631 |
Return a lookup usable for doing outerq.filter(lookup=self) and a |
| 2632 |
boolean indicating if the joins in the prefix contain a LEFT OUTER join. |
| 2633 |
_""" |
| 2634 |
all_paths = [] |
| 2635 |
for _, paths in names_with_path: |
| 2636 |
all_paths.extend(paths) |
| 2637 |
contains_louter = False |
| 2638 |
# Trim and operate only on tables that were generated for |
| 2639 |
# the lookup part of the query. That is, avoid trimming |
| 2640 |
# joins generated for F() expressions. |
| 2641 |
lookup_tables = [ |
| 2642 |
t for t in self.alias_map if t in self._lookup_joins or t == self.base_table |
| 2643 |
] |
| 2644 |
for trimmed_paths, path in enumerate(all_paths): |
| 2645 |
if path.m2m: |
| 2646 |
break |
| 2647 |
if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER: |
| 2648 |
contains_louter = True |
| 2649 |
alias = lookup_tables[trimmed_paths] |
| 2650 |
self.unref_alias(alias) |
| 2651 |
# The path.join_field is a Rel, lets get the other side's field |
| 2652 |
join_field = path.join_field.field |
| 2653 |
# Build the filter prefix. |
| 2654 |
paths_in_prefix = trimmed_paths |
| 2655 |
trimmed_prefix = [] |
| 2656 |
for name, path in names_with_path: |
| 2657 |
if paths_in_prefix - len(path) < 0: |
| 2658 |
break |
| 2659 |
trimmed_prefix.append(name) |
| 2660 |
paths_in_prefix -= len(path) |
| 2661 |
trimmed_prefix.append(join_field.foreign_related_fields[0].name) |
| 2662 |
trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix) |
| 2663 |
# Lets still see if we can trim the first join from the inner query |
| 2664 |
# (that is, self). We can't do this for: |
| 2665 |
# - LEFT JOINs because we would miss those rows that have nothing on |
| 2666 |
# the outer side, |
| 2667 |
# - INNER JOINs from filtered relations because we would miss their |
| 2668 |
# filters. |
| 2669 |
first_join = self.alias_map[lookup_tables[trimmed_paths + 1]] |
| 2670 |
if first_join.join_type != LOUTER and not first_join.filtered_relation: |
| 2671 |
select_fields = [r[0] for r in join_field.related_fields] |
| 2672 |
select_alias = lookup_tables[trimmed_paths + 1] |
| 2673 |
self.unref_alias(lookup_tables[trimmed_paths]) |
| 2674 |
extra_restriction = join_field.get_extra_restriction( |
| 2675 |
None, lookup_tables[trimmed_paths + 1] |
| 2676 |
) |
| 2677 |
if extra_restriction: |
| 2678 |
self.where.add(extra_restriction, AND) |
| 2679 |
else: |
| 2680 |
# TODO: It might be possible to trim more joins from the start of the |
| 2681 |
# inner query if it happens to have a longer join chain containing the |
| 2682 |
# values in select_fields. Lets punt this one for now. |
| 2683 |
select_fields = [r[1] for r in join_field.related_fields] |
| 2684 |
select_alias = lookup_tables[trimmed_paths] |
| 2685 |
# The found starting point is likely a join_class instead of a |
| 2686 |
# base_table_class reference. But the first entry in the query's FROM |
| 2687 |
# clause must not be a JOIN. |
| 2688 |
for table in self.alias_map: |
| 2689 |
if self.alias_refcount[table] > 0: |
| 2690 |
self.alias_map[table] = self.base_table_class( |
| 2691 |
self.alias_map[table].table_name, |
| 2692 |
table, |
| 2693 |
) |
| 2694 |
break |
| 2695 |
self.set_select([f.get_col(select_alias) for f in select_fields]) |
| 2696 |
return trimmed_prefix, contains_louter |
| 2697 |
|
| 2698 |
def is_nullable(self, field): |
| 2699 |
""" |
| 2700 |
Check if the given field should be treated as nullable. |
| 2701 |
|
| 2702 |
Some backends treat '' as null and Django treats such fields as |
| 2703 |
nullable for those backends. In such situations field.null can be |
| 2704 |
False even if we should treat the field as nullable. |
| 2705 |
""" |
| 2706 |
# We need to use DEFAULT_DB_ALIAS here, as QuerySet does not have |
| 2707 |
# (nor should it have) knowledge of which connection is going to be |
| 2708 |
# used. The proper fix would be to defer all decisions where |
| 2709 |
# is_nullable() is needed to the compiler stage, but that is not easy |
| 2710 |
# to do currently. |
| 2711 |
return field.null or ( |
| 2712 |
field.empty_strings_allowed |
| 2713 |
and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls |
| 2714 |
) |
| 2715 |
|
| 2716 |
|
| 2717 |
def get_order_dir(field, default="ASC"): |
| 2718 |
""" |
| 2719 |
Return the field name and direction for an order specification. For |
| 2720 |
example, '-foo' is returned as ('foo', 'DESC'). |
| 2721 |
|
| 2722 |
The 'default' param is used to indicate which way no prefix (or a '+' |
| 2723 |
prefix) should sort. The '-' prefix always sorts the opposite way. |
| 2724 |
""" |
| 2725 |
dirn = ORDER_DIR[default] |
| 2726 |
if field[0] == "-": |
| 2727 |
return field[1:], dirn[1] |
| 2728 |
return field, dirn[0] |
| 2729 |
|
| 2730 |
|
| 2731 |
class JoinPromoter: |
| 2732 |
""" |
| 2733 |
A class to abstract away join promotion problems for complex filter |
| 2734 |
conditions. |
| 2735 |
""" |
| 2736 |
|
| 2737 |
def __init__(self, connector, num_children, negated): |
| 2738 |
self.connector = connector |
| 2739 |
self.negated = negated |
| 2740 |
if self.negated: |
| 2741 |
if connector == AND: |
| 2742 |
self.effective_connector = OR |
| 2743 |
else: |
| 2744 |
self.effective_connector = AND |
| 2745 |
else: |
| 2746 |
self.effective_connector = self.connector |
| 2747 |
self.num_children = num_children |
| 2748 |
# Maps of table alias to how many times it is seen as required for |
| 2749 |
# inner and/or outer joins. |
| 2750 |
self.votes = Counter() |
| 2751 |
|
| 2752 |
def __repr__(self): |
| 2753 |
return ( |
| 2754 |
f"{self.__class__.__qualname__}(connector={self.connector!r}, " |
| 2755 |
f"num_children={self.num_children!r}, negated={self.negated!r})" |
| 2756 |
) |
| 2757 |
|
| 2758 |
def add_votes(self, votes): |
| 2759 |
""" |
| 2760 |
Add single vote per item to self.votes. Parameter can be any |
| 2761 |
iterable. |
| 2762 |
""" |
| 2763 |
self.votes.update(votes) |
| 2764 |
|
| 2765 |
def update_join_types(self, query): |
| 2766 |
""" |
| 2767 |
Change join types so that the generated query is as efficient as |
| 2768 |
possible, but still correct. So, change as many joins as possible |
| 2769 |
to INNER, but don't make OUTER joins INNER if that could remove |
| 2770 |
results from the query. |
| 2771 |
""" |
| 2772 |
to_promote = set() |
| 2773 |
to_demote = set() |
| 2774 |
# The effective_connector is used so that NOT (a AND b) is treated |
| 2775 |
# similarly to (a OR b) for join promotion. |
| 2776 |
for table, votes in self.votes.items(): |
| 2777 |
# We must use outer joins in OR case when the join isn't contained |
| 2778 |
# in all of the joins. Otherwise the INNER JOIN itself could remove |
| 2779 |
# valid results. Consider the case where a model with rel_a and |
| 2780 |
# rel_b relations is queried with rel_a__col=1 | rel_b__col=2. Now, |
| 2781 |
# if rel_a join doesn't produce any results is null (for example |
| 2782 |
# reverse foreign key or null value in direct foreign key), and |
| 2783 |
# there is a matching row in rel_b with col=2, then an INNER join |
| 2784 |
# to rel_a would remove a valid match from the query. So, we need |
| 2785 |
# to promote any existing INNER to LOUTER (it is possible this |
| 2786 |
# promotion in turn will be demoted later on). |
| 2787 |
if self.effective_connector == OR and votes < self.num_children: |
| 2788 |
to_promote.add(table) |
| 2789 |
# If connector is AND and there is a filter that can match only |
| 2790 |
# when there is a joinable row, then use INNER. For example, in |
| 2791 |
# rel_a__col=1 & rel_b__col=2, if either of the rels produce NULL |
| 2792 |
# as join output, then the col=1 or col=2 can't match (as |
| 2793 |
# NULL=anything is always false). |
| 2794 |
# For the OR case, if all children voted for a join to be inner, |
| 2795 |
# then we can use INNER for the join. For example: |
| 2796 |
# (rel_a__col__icontains=Alex | rel_a__col__icontains=Russell) |
| 2797 |
# then if rel_a doesn't produce any rows, the whole condition |
| 2798 |
# can't match. Hence we can safely use INNER join. |
| 2799 |
if self.effective_connector == AND or ( |
| 2800 |
self.effective_connector == OR and votes == self.num_children |
| 2801 |
): |
| 2802 |
to_demote.add(table) |
| 2803 |
# Finally, what happens in cases where we have: |
| 2804 |
# (rel_a__col=1|rel_b__col=2) & rel_a__col__gte=0 |
| 2805 |
# Now, we first generate the OR clause, and promote joins for it |
| 2806 |
# in the first if branch above. Both rel_a and rel_b are promoted |
| 2807 |
# to LOUTER joins. After that we do the AND case. The OR case |
| 2808 |
# voted no inner joins but the rel_a__col__gte=0 votes inner join |
| 2809 |
# for rel_a. We demote it back to INNER join (in AND case a single |
| 2810 |
# vote is enough). The demotion is OK, if rel_a doesn't produce |
| 2811 |
# rows, then the rel_a__col__gte=0 clause can't be true, and thus |
| 2812 |
# the whole clause must be false. So, it is safe to use INNER |
| 2813 |
# join. |
| 2814 |
# Note that in this example we could just as well have the __gte |
| 2815 |
# clause and the OR clause swapped. Or we could replace the __gte |
| 2816 |
# clause with an OR clause containing rel_a__col=1|rel_a__col=2, |
| 2817 |
# and again we could safely demote to INNER. |
| 2818 |
query.promote_joins(to_promote) |
| 2819 |
query.demote_joins(to_demote) |
| 2820 |
return to_demote |