Debian Codegraph @

django/db/models/query.py

raw
1 """
2 The main QuerySet implementation. This provides the public API for the ORM.
3 """
4
5 import copy
6 import operator
7 import warnings
8 from itertools import chain, islice
9
10 from asgiref.sync import sync_to_async
11
12 import django
13 from django.conf import settings
14 from django.core import exceptions
15 from django.db import (
16 DJANGO_VERSION_PICKLE_KEY,
17 IntegrityError,
18 NotSupportedError,
19 connections,
20 router,
21 transaction,
22 )
23 from django.db.models import AutoField, DateField, DateTimeField, Field, sql
24 from django.db.models.constants import LOOKUP_SEP, OnConflict
25 from django.db.models.deletion import Collector
26 from django.db.models.expressions import Case, F, Value, When
27 from django.db.models.functions import Cast, Trunc
28 from django.db.models.query_utils import FilteredRelation, Q
29 from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE, ROW_COUNT
30 from django.db.models.utils import (
31 AltersData,
32 create_namedtuple_class,
33 resolve_callables,
34 )
35 from django.utils import timezone
36 from django.utils.deprecation import RemovedInDjango60Warning
37 from django.utils.functional import cached_property, partition
38
39 # The maximum number of results to fetch in a get() query.
40 MAX_GET_RESULTS = 21
41
42 # The maximum number of items to display in a QuerySet.__repr__
43 REPR_OUTPUT_SIZE = 20
44
45 PROHIBITED_FILTER_KWARGS = frozenset(["_connector", "_negated"])
46
47
48 class BaseIterable:
49 def __init__(
50 self, queryset, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE
51 ):
52 self.queryset = queryset
53 self.chunked_fetch = chunked_fetch
54 self.chunk_size = chunk_size
55
56 async def _async_generator(self):
57 # Generators don't actually start running until the first time you call
58 # next() on them, so make the generator object in the async thread and
59 # then repeatedly dispatch to it in a sync thread.
60 sync_generator = self.__iter__()
61
62 def next_slice(gen):
63 return list(islice(gen, self.chunk_size))
64
65 while True:
66 chunk = await sync_to_async(next_slice)(sync_generator)
67 for item in chunk:
68 yield item
69 if len(chunk) < self.chunk_size:
70 break
71
72 # __aiter__() is a *synchronous* method that has to then return an
73 # *asynchronous* iterator/generator. Thus, nest an async generator inside
74 # it.
75 # This is a generic iterable converter for now, and is going to suffer a
76 # performance penalty on large sets of items due to the cost of crossing
77 # over the sync barrier for each chunk. Custom __aiter__() methods should
78 # be added to each Iterable subclass, but that needs some work in the
79 # Compiler first.
80 def __aiter__(self):
81 return self._async_generator()
82
83
84 class ModelIterable(BaseIterable):
85 """Iterable that yields a model instance for each row."""
86
87 def __iter__(self):
88 queryset = self.queryset
89 db = queryset.db
90 compiler = queryset.query.get_compiler(using=db)
91 # Execute the query. This will also fill compiler.select, klass_info,
92 # and annotations.
93 results = compiler.execute_sql(
94 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
95 )
96 select, klass_info, annotation_col_map = (
97 compiler.select,
98 compiler.klass_info,
99 compiler.annotation_col_map,
100 )
101 model_cls = klass_info["model"]
102 select_fields = klass_info["select_fields"]
103 model_fields_start, model_fields_end = select_fields[0], select_fields[-1] + 1
104 init_list = [
105 f[0].target.attname for f in select[model_fields_start:model_fields_end]
106 ]
107 related_populators = get_related_populators(klass_info, select, db)
108 known_related_objects = [
109 (
110 field,
111 related_objs,
112 operator.attrgetter(
113 *[
114 (
115 field.attname
116 if from_field == "self"
117 else queryset.model._meta.get_field(from_field).attname
118 )
119 for from_field in field.from_fields
120 ]
121 ),
122 )
123 for field, related_objs in queryset._known_related_objects.items()
124 ]
125 for row in compiler.results_iter(results):
126 obj = model_cls.from_db(
127 db, init_list, row[model_fields_start:model_fields_end]
128 )
129 for rel_populator in related_populators:
130 rel_populator.populate(row, obj)
131 if annotation_col_map:
132 for attr_name, col_pos in annotation_col_map.items():
133 setattr(obj, attr_name, row[col_pos])
134
135 # Add the known related objects to the model.
136 for field, rel_objs, rel_getter in known_related_objects:
137 # Avoid overwriting objects loaded by, e.g., select_related().
138 if field.is_cached(obj):
139 continue
140 rel_obj_id = rel_getter(obj)
141 try:
142 rel_obj = rel_objs[rel_obj_id]
143 except KeyError:
144 pass # May happen in qs1 | qs2 scenarios.
145 else:
146 setattr(obj, field.name, rel_obj)
147
148 yield obj
149
150
151 class RawModelIterable(BaseIterable):
152 """
153 Iterable that yields a model instance for each row from a raw queryset.
154 """
155
156 def __iter__(self):
157 # Cache some things for performance reasons outside the loop.
158 db = self.queryset.db
159 query = self.queryset.query
160 connection = connections[db]
161 compiler = connection.ops.compiler("SQLCompiler")(query, connection, db)
162 query_iterator = iter(query)
163
164 try:
165 (
166 model_init_names,
167 model_init_pos,
168 annotation_fields,
169 ) = self.queryset.resolve_model_init_order()
170 model_cls = self.queryset.model
171 if model_cls._meta.pk.attname not in model_init_names:
172 raise exceptions.FieldDoesNotExist(
173 "Raw query must include the primary key"
174 )
175 fields = [self.queryset.model_fields.get(c) for c in self.queryset.columns]
176 cols = [f.get_col(f.model._meta.db_table) if f else None for f in fields]
177 converters = compiler.get_converters(cols)
178 if converters:
179 query_iterator = compiler.apply_converters(query_iterator, converters)
180 if compiler.has_composite_fields(cols):
181 query_iterator = compiler.composite_fields_to_tuples(
182 query_iterator, cols
183 )
184 for values in query_iterator:
185 # Associate fields to values
186 model_init_values = [values[pos] for pos in model_init_pos]
187 instance = model_cls.from_db(db, model_init_names, model_init_values)
188 if annotation_fields:
189 for column, pos in annotation_fields:
190 setattr(instance, column, values[pos])
191 yield instance
192 finally:
193 # Done iterating the Query. If it has its own cursor, close it.
194 if hasattr(query, "cursor") and query.cursor:
195 query.cursor.close()
196
197
198 class ValuesIterable(BaseIterable):
199 """
200 Iterable returned by QuerySet.values() that yields a dict for each row.
201 """
202
203 def __iter__(self):
204 queryset = self.queryset
205 query = queryset.query
206 compiler = query.get_compiler(queryset.db)
207
208 if query.selected:
209 names = list(query.selected)
210 else:
211 # extra(select=...) cols are always at the start of the row.
212 names = [
213 *query.extra_select,
214 *query.values_select,
215 *query.annotation_select,
216 ]
217 indexes = range(len(names))
218 for row in compiler.results_iter(
219 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
220 ):
221 yield {names[i]: row[i] for i in indexes}
222
223
224 class ValuesListIterable(BaseIterable):
225 """
226 Iterable returned by QuerySet.values_list(flat=False) that yields a tuple
227 for each row.
228 """
229
230 def __iter__(self):
231 queryset = self.queryset
232 query = queryset.query
233 compiler = query.get_compiler(queryset.db)
234 return compiler.results_iter(
235 tuple_expected=True,
236 chunked_fetch=self.chunked_fetch,
237 chunk_size=self.chunk_size,
238 )
239
240
241 class NamedValuesListIterable(ValuesListIterable):
242 """
243 Iterable returned by QuerySet.values_list(named=True) that yields a
244 namedtuple for each row.
245 """
246
247 def __iter__(self):
248 queryset = self.queryset
249 if queryset._fields:
250 names = queryset._fields
251 else:
252 query = queryset.query
253 names = [
254 *query.extra_select,
255 *query.values_select,
256 *query.annotation_select,
257 ]
258 tuple_class = create_namedtuple_class(*names)
259 new = tuple.__new__
260 for row in super().__iter__():
261 yield new(tuple_class, row)
262
263
264 class FlatValuesListIterable(BaseIterable):
265 """
266 Iterable returned by QuerySet.values_list(flat=True) that yields single
267 values.
268 """
269
270 def __iter__(self):
271 queryset = self.queryset
272 compiler = queryset.query.get_compiler(queryset.db)
273 for row in compiler.results_iter(
274 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
275 ):
276 yield row[0]
277
278
279 class QuerySet(AltersData):
280 """Represent a lazy database lookup for a set of objects."""
281
282 def __init__(self, model=None, query=None, using=None, hints=None):
283 self.model = model
284 self._db = using
285 self._hints = hints or {}
286 self._query = query or sql.Query(self.model)
287 self._result_cache = None
288 self._sticky_filter = False
289 self._for_write = False
290 self._prefetch_related_lookups = ()
291 self._prefetch_done = False
292 self._known_related_objects = {} # {rel_field: {pk: rel_obj}}
293 self._iterable_class = ModelIterable
294 self._fields = None
295 self._defer_next_filter = False
296 self._deferred_filter = None
297
298 @property
299 def query(self):
300 if self._deferred_filter:
301 negate, args, kwargs = self._deferred_filter
302 self._filter_or_exclude_inplace(negate, args, kwargs)
303 self._deferred_filter = None
304 return self._query
305
306 @query.setter
307 def query(self, value):
308 if value.values_select:
309 self._iterable_class = ValuesIterable
310 self._query = value
311
312 def as_manager(cls):
313 # Address the circular dependency between `Queryset` and `Manager`.
314 from django.db.models.manager import Manager
315
316 manager = Manager.from_queryset(cls)()
317 manager._built_with_as_manager = True
318 return manager
319
320 as_manager.queryset_only = True
321 as_manager = classmethod(as_manager)
322
323 ########################
324 # PYTHON MAGIC METHODS #
325 ########################
326
327 def __deepcopy__(self, memo):
328 """Don't populate the QuerySet's cache."""
329 obj = self.__class__()
330 for k, v in self.__dict__.items():
331 if k == "_result_cache":
332 obj.__dict__[k] = None
333 else:
334 obj.__dict__[k] = copy.deepcopy(v, memo)
335 return obj
336
337 def __getstate__(self):
338 # Force the cache to be fully populated.
339 self._fetch_all()
340 return {**self.__dict__, DJANGO_VERSION_PICKLE_KEY: django.__version__}
341
342 def __setstate__(self, state):
343 pickled_version = state.get(DJANGO_VERSION_PICKLE_KEY)
344 if pickled_version:
345 if pickled_version != django.__version__:
346 warnings.warn(
347 "Pickled queryset instance's Django version %s does not "
348 "match the current version %s."
349 % (pickled_version, django.__version__),
350 RuntimeWarning,
351 stacklevel=2,
352 )
353 else:
354 warnings.warn(
355 "Pickled queryset instance's Django version is not specified.",
356 RuntimeWarning,
357 stacklevel=2,
358 )
359 self.__dict__.update(state)
360
361 def __repr__(self):
362 data = list(self[: REPR_OUTPUT_SIZE + 1])
363 if len(data) > REPR_OUTPUT_SIZE:
364 data[-1] = "...(remaining elements truncated)..."
365 return "<%s %r>" % (self.__class__.__name__, data)
366
367 def __len__(self):
368 self._fetch_all()
369 return len(self._result_cache)
370
371 def __iter__(self):
372 """
373 The queryset iterator protocol uses three nested iterators in the
374 default case:
375 1. sql.compiler.execute_sql()
376 - Returns 100 rows at time (constants.GET_ITERATOR_CHUNK_SIZE)
377 using cursor.fetchmany(). This part is responsible for
378 doing some column masking, and returning the rows in chunks.
379 2. sql.compiler.results_iter()
380 - Returns one row at time. At this point the rows are still just
381 tuples. In some cases the return values are converted to
382 Python values at this location.
383 3. self.iterator()
384 - Responsible for turning the rows into model objects.
385 """
386 self._fetch_all()
387 return iter(self._result_cache)
388
389 def __aiter__(self):
390 # Remember, __aiter__ itself is synchronous, it's the thing it returns
391 # that is async!
392 async def generator():
393 await sync_to_async(self._fetch_all)()
394 for item in self._result_cache:
395 yield item
396
397 return generator()
398
399 def __bool__(self):
400 self._fetch_all()
401 return bool(self._result_cache)
402
403 def __getitem__(self, k):
404 """Retrieve an item or slice from the set of results."""
405 if not isinstance(k, (int, slice)):
406 raise TypeError(
407 "QuerySet indices must be integers or slices, not %s."
408 % type(k).__name__
409 )
410 if (isinstance(k, int) and k < 0) or (
411 isinstance(k, slice)
412 and (
413 (k.start is not None and k.start < 0)
414 or (k.stop is not None and k.stop < 0)
415 )
416 ):
417 raise ValueError("Negative indexing is not supported.")
418
419 if self._result_cache is not None:
420 return self._result_cache[k]
421
422 if isinstance(k, slice):
423 qs = self._chain()
424 if k.start is not None:
425 start = int(k.start)
426 else:
427 start = None
428 if k.stop is not None:
429 stop = int(k.stop)
430 else:
431 stop = None
432 qs.query.set_limits(start, stop)
433 return list(qs)[:: k.step] if k.step else qs
434
435 qs = self._chain()
436 qs.query.set_limits(k, k + 1)
437 qs._fetch_all()
438 return qs._result_cache[0]
439
440 def __class_getitem__(cls, *args, **kwargs):
441 return cls
442
443 def __and__(self, other):
444 self._check_operator_queryset(other, "&")
445 self._merge_sanity_check(other)
446 if isinstance(other, EmptyQuerySet):
447 return other
448 if isinstance(self, EmptyQuerySet):
449 return self
450 combined = self._chain()
451 combined._merge_known_related_objects(other)
452 combined.query.combine(other.query, sql.AND)
453 return combined
454
455 def __or__(self, other):
456 self._check_operator_queryset(other, "|")
457 self._merge_sanity_check(other)
458 if isinstance(self, EmptyQuerySet):
459 return other
460 if isinstance(other, EmptyQuerySet):
461 return self
462 query = (
463 self
464 if self.query.can_filter()
465 else self.model._base_manager.filter(pk__in=self.values("pk"))
466 )
467 combined = query._chain()
468 combined._merge_known_related_objects(other)
469 if not other.query.can_filter():
470 other = other.model._base_manager.filter(pk__in=other.values("pk"))
471 combined.query.combine(other.query, sql.OR)
472 return combined
473
474 def __xor__(self, other):
475 self._check_operator_queryset(other, "^")
476 self._merge_sanity_check(other)
477 if isinstance(self, EmptyQuerySet):
478 return other
479 if isinstance(other, EmptyQuerySet):
480 return self
481 query = (
482 self
483 if self.query.can_filter()
484 else self.model._base_manager.filter(pk__in=self.values("pk"))
485 )
486 combined = query._chain()
487 combined._merge_known_related_objects(other)
488 if not other.query.can_filter():
489 other = other.model._base_manager.filter(pk__in=other.values("pk"))
490 combined.query.combine(other.query, sql.XOR)
491 return combined
492
493 ####################################
494 # METHODS THAT DO DATABASE QUERIES #
495 ####################################
496
497 def _iterator(self, use_chunked_fetch, chunk_size):
498 iterable = self._iterable_class(
499 self,
500 chunked_fetch=use_chunked_fetch,
501 chunk_size=chunk_size or 2000,
502 )
503 if not self._prefetch_related_lookups or chunk_size is None:
504 yield from iterable
505 return
506
507 iterator = iter(iterable)
508 while results := list(islice(iterator, chunk_size)):
509 prefetch_related_objects(results, *self._prefetch_related_lookups)
510 yield from results
511
512 def iterator(self, chunk_size=None):
513 """
514 An iterator over the results from applying this QuerySet to the
515 database. chunk_size must be provided for QuerySets that prefetch
516 related objects. Otherwise, a default chunk_size of 2000 is supplied.
517 """
518 if chunk_size is None:
519 if self._prefetch_related_lookups:
520 raise ValueError(
521 "chunk_size must be provided when using QuerySet.iterator() after "
522 "prefetch_related()."
523 )
524 elif chunk_size <= 0:
525 raise ValueError("Chunk size must be strictly positive.")
526 use_chunked_fetch = not connections[self.db].settings_dict.get(
527 "DISABLE_SERVER_SIDE_CURSORS"
528 )
529 return self._iterator(use_chunked_fetch, chunk_size)
530
531 async def aiterator(self, chunk_size=2000):
532 """
533 An asynchronous iterator over the results from applying this QuerySet
534 to the database.
535 """
536 if chunk_size <= 0:
537 raise ValueError("Chunk size must be strictly positive.")
538 use_chunked_fetch = not connections[self.db].settings_dict.get(
539 "DISABLE_SERVER_SIDE_CURSORS"
540 )
541 iterable = self._iterable_class(
542 self, chunked_fetch=use_chunked_fetch, chunk_size=chunk_size
543 )
544 if self._prefetch_related_lookups:
545 results = []
546
547 async for item in iterable:
548 results.append(item)
549 if len(results) >= chunk_size:
550 await aprefetch_related_objects(
551 results, *self._prefetch_related_lookups
552 )
553 for result in results:
554 yield result
555 results.clear()
556
557 if results:
558 await aprefetch_related_objects(
559 results, *self._prefetch_related_lookups
560 )
561 for result in results:
562 yield result
563 else:
564 async for item in iterable:
565 yield item
566
567 def aggregate(self, *args, **kwargs):
568 """
569 Return a dictionary containing the calculations (aggregation)
570 over the current queryset.
571
572 If args is present the expression is passed as a kwarg using
573 the Aggregate object's default alias.
574 """
575 if self.query.distinct_fields:
576 raise NotImplementedError("aggregate() + distinct(fields) not implemented.")
577 self._validate_values_are_expressions(
578 (*args, *kwargs.values()), method_name="aggregate"
579 )
580 for arg in args:
581 # The default_alias property raises TypeError if default_alias
582 # can't be set automatically or AttributeError if it isn't an
583 # attribute.
584 try:
585 arg.default_alias
586 except (AttributeError, TypeError):
587 raise TypeError("Complex aggregates require an alias")
588 kwargs[arg.default_alias] = arg
589
590 return self.query.chain().get_aggregation(self.db, kwargs)
591
592 async def aaggregate(self, *args, **kwargs):
593 return await sync_to_async(self.aggregate)(*args, **kwargs)
594
595 def count(self):
596 """
597 Perform a SELECT COUNT() and return the number of records as an
598 integer.
599
600 If the QuerySet is already fully cached, return the length of the
601 cached results set to avoid multiple SELECT COUNT(*) calls.
602 """
603 if self._result_cache is not None:
604 return len(self._result_cache)
605
606 return self.query.get_count(using=self.db)
607
608 async def acount(self):
609 return await sync_to_async(self.count)()
610
611 def get(self, *args, **kwargs):
612 """
613 Perform the query and return a single object matching the given
614 keyword arguments.
615 """
616 if self.query.combinator and (args or kwargs):
617 raise NotSupportedError(
618 "Calling QuerySet.get(...) with filters after %s() is not "
619 "supported." % self.query.combinator
620 )
621 clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
622 if self.query.can_filter() and not self.query.distinct_fields:
623 clone = clone.order_by()
624 limit = None
625 if (
626 not clone.query.select_for_update
627 or connections[clone.db].features.supports_select_for_update_with_limit
628 ):
629 limit = MAX_GET_RESULTS
630 clone.query.set_limits(high=limit)
631 num = len(clone)
632 if num == 1:
633 return clone._result_cache[0]
634 if not num:
635 raise self.model.DoesNotExist(
636 "%s matching query does not exist." % self.model._meta.object_name
637 )
638 raise self.model.MultipleObjectsReturned(
639 "get() returned more than one %s -- it returned %s!"
640 % (
641 self.model._meta.object_name,
642 num if not limit or num < limit else "more than %s" % (limit - 1),
643 )
644 )
645
646 async def aget(self, *args, **kwargs):
647 return await sync_to_async(self.get)(*args, **kwargs)
648
649 def create(self, **kwargs):
650 """
651 Create a new object with the given kwargs, saving it to the database
652 and returning the created object.
653 """
654 reverse_one_to_one_fields = frozenset(kwargs).intersection(
655 self.model._meta._reverse_one_to_one_field_names
656 )
657 if reverse_one_to_one_fields:
658 raise ValueError(
659 "The following fields do not exist in this model: %s"
660 % ", ".join(reverse_one_to_one_fields)
661 )
662
663 obj = self.model(**kwargs)
664 self._for_write = True
665 obj.save(force_insert=True, using=self.db)
666 return obj
667
668 create.alters_data = True
669
670 async def acreate(self, **kwargs):
671 return await sync_to_async(self.create)(**kwargs)
672
673 acreate.alters_data = True
674
675 def _prepare_for_bulk_create(self, objs):
676 from django.db.models.expressions import DatabaseDefault
677
678 connection = connections[self.db]
679 for obj in objs:
680 if not obj._is_pk_set():
681 # Populate new PK values.
682 obj.pk = obj._meta.pk.get_pk_value_on_save(obj)
683 if not connection.features.supports_default_keyword_in_bulk_insert:
684 for field in obj._meta.fields:
685 if field.generated:
686 continue
687 value = getattr(obj, field.attname)
688 if isinstance(value, DatabaseDefault):
689 setattr(obj, field.attname, field.db_default)
690
691 obj._prepare_related_fields_for_save(operation_name="bulk_create")
692
693 def _check_bulk_create_options(
694 self, ignore_conflicts, update_conflicts, update_fields, unique_fields
695 ):
696 if ignore_conflicts and update_conflicts:
697 raise ValueError(
698 "ignore_conflicts and update_conflicts are mutually exclusive."
699 )
700 db_features = connections[self.db].features
701 if ignore_conflicts:
702 if not db_features.supports_ignore_conflicts:
703 raise NotSupportedError(
704 "This database backend does not support ignoring conflicts."
705 )
706 return OnConflict.IGNORE
707 elif update_conflicts:
708 if not db_features.supports_update_conflicts:
709 raise NotSupportedError(
710 "This database backend does not support updating conflicts."
711 )
712 if not update_fields:
713 raise ValueError(
714 "Fields that will be updated when a row insertion fails "
715 "on conflicts must be provided."
716 )
717 if unique_fields and not db_features.supports_update_conflicts_with_target:
718 raise NotSupportedError(
719 "This database backend does not support updating "
720 "conflicts with specifying unique fields that can trigger "
721 "the upsert."
722 )
723 if not unique_fields and db_features.supports_update_conflicts_with_target:
724 raise ValueError(
725 "Unique fields that can trigger the upsert must be provided."
726 )
727 # Updating primary keys and non-concrete fields is forbidden.
728 if any(not f.concrete or f.many_to_many for f in update_fields):
729 raise ValueError(
730 "bulk_create() can only be used with concrete fields in "
731 "update_fields."
732 )
733 if any(f in self.model._meta.pk_fields for f in update_fields):
734 raise ValueError(
735 "bulk_create() cannot be used with primary keys in "
736 "update_fields."
737 )
738 if unique_fields:
739 if any(not f.concrete or f.many_to_many for f in unique_fields):
740 raise ValueError(
741 "bulk_create() can only be used with concrete fields "
742 "in unique_fields."
743 )
744 return OnConflict.UPDATE
745 return None
746
747 def bulk_create(
748 self,
749 objs,
750 batch_size=None,
751 ignore_conflicts=False,
752 update_conflicts=False,
753 update_fields=None,
754 unique_fields=None,
755 ):
756 """
757 Insert each of the instances into the database. Do *not* call
758 save() on each of the instances, do not send any pre/post_save
759 signals, and do not set the primary key attribute if it is an
760 autoincrement field (except if features.can_return_rows_from_bulk_insert=True).
761 Multi-table models are not supported.
762 """
763 # When you bulk insert you don't get the primary keys back (if it's an
764 # autoincrement, except if can_return_rows_from_bulk_insert=True), so
765 # you can't insert into the child tables which references this. There
766 # are two workarounds:
767 # 1) This could be implemented if you didn't have an autoincrement pk
768 # 2) You could do it by doing O(n) normal inserts into the parent
769 # tables to get the primary keys back and then doing a single bulk
770 # insert into the childmost table.
771 # We currently set the primary keys on the objects when using
772 # PostgreSQL via the RETURNING ID clause. It should be possible for
773 # Oracle as well, but the semantics for extracting the primary keys is
774 # trickier so it's not done yet.
775 if batch_size is not None and batch_size <= 0:
776 raise ValueError("Batch size must be a positive integer.")
777 # Check that the parents share the same concrete model with the our
778 # model to detect the inheritance pattern ConcreteGrandParent ->
779 # MultiTableParent -> ProxyChild. Simply checking self.model._meta.proxy
780 # would not identify that case as involving multiple tables.
781 for parent in self.model._meta.all_parents:
782 if parent._meta.concrete_model is not self.model._meta.concrete_model:
783 raise ValueError("Can't bulk create a multi-table inherited model")
784 if not objs:
785 return objs
786 opts = self.model._meta
787 if unique_fields:
788 # Primary key is allowed in unique_fields.
789 unique_fields = [
790 self.model._meta.get_field(opts.pk.name if name == "pk" else name)
791 for name in unique_fields
792 ]
793 if update_fields:
794 update_fields = [self.model._meta.get_field(name) for name in update_fields]
795 on_conflict = self._check_bulk_create_options(
796 ignore_conflicts,
797 update_conflicts,
798 update_fields,
799 unique_fields,
800 )
801 self._for_write = True
802 fields = [f for f in opts.concrete_fields if not f.generated]
803 objs = list(objs)
804 self._prepare_for_bulk_create(objs)
805 with transaction.atomic(using=self.db, savepoint=False):
806 objs_without_pk, objs_with_pk = partition(lambda o: o._is_pk_set(), objs)
807 if objs_with_pk:
808 returned_columns = self._batched_insert(
809 objs_with_pk,
810 fields,
811 batch_size,
812 on_conflict=on_conflict,
813 update_fields=update_fields,
814 unique_fields=unique_fields,
815 )
816 for obj_with_pk, results in zip(objs_with_pk, returned_columns):
817 for result, field in zip(results, opts.db_returning_fields):
818 if field != opts.pk:
819 setattr(obj_with_pk, field.attname, result)
820 for obj_with_pk in objs_with_pk:
821 obj_with_pk._state.adding = False
822 obj_with_pk._state.db = self.db
823 if objs_without_pk:
824 fields = [f for f in fields if not isinstance(f, AutoField)]
825 returned_columns = self._batched_insert(
826 objs_without_pk,
827 fields,
828 batch_size,
829 on_conflict=on_conflict,
830 update_fields=update_fields,
831 unique_fields=unique_fields,
832 )
833 connection = connections[self.db]
834 if (
835 connection.features.can_return_rows_from_bulk_insert
836 and on_conflict is None
837 ):
838 assert len(returned_columns) == len(objs_without_pk)
839 for obj_without_pk, results in zip(objs_without_pk, returned_columns):
840 for result, field in zip(results, opts.db_returning_fields):
841 setattr(obj_without_pk, field.attname, result)
842 obj_without_pk._state.adding = False
843 obj_without_pk._state.db = self.db
844
845 return objs
846
847 bulk_create.alters_data = True
848
849 async def abulk_create(
850 self,
851 objs,
852 batch_size=None,
853 ignore_conflicts=False,
854 update_conflicts=False,
855 update_fields=None,
856 unique_fields=None,
857 ):
858 return await sync_to_async(self.bulk_create)(
859 objs=objs,
860 batch_size=batch_size,
861 ignore_conflicts=ignore_conflicts,
862 update_conflicts=update_conflicts,
863 update_fields=update_fields,
864 unique_fields=unique_fields,
865 )
866
867 abulk_create.alters_data = True
868
869 def bulk_update(self, objs, fields, batch_size=None):
870 """
871 Update the given fields in each of the given objects in the database.
872 """
873 if batch_size is not None and batch_size <= 0:
874 raise ValueError("Batch size must be a positive integer.")
875 if not fields:
876 raise ValueError("Field names must be given to bulk_update().")
877 objs = tuple(objs)
878 if not all(obj._is_pk_set() for obj in objs):
879 raise ValueError("All bulk_update() objects must have a primary key set.")
880 opts = self.model._meta
881 fields = [opts.get_field(name) for name in fields]
882 if any(not f.concrete or f.many_to_many for f in fields):
883 raise ValueError("bulk_update() can only be used with concrete fields.")
884 all_pk_fields = set(opts.pk_fields)
885 for parent in opts.all_parents:
886 all_pk_fields.update(parent._meta.pk_fields)
887 if any(f in all_pk_fields for f in fields):
888 raise ValueError("bulk_update() cannot be used with primary key fields.")
889 if not objs:
890 return 0
891 for obj in objs:
892 obj._prepare_related_fields_for_save(
893 operation_name="bulk_update", fields=fields
894 )
895 # PK is used twice in the resulting update query, once in the filter
896 # and once in the WHEN. Each field will also have one CAST.
897 self._for_write = True
898 connection = connections[self.db]
899 max_batch_size = connection.ops.bulk_batch_size(
900 [opts.pk, opts.pk] + fields, objs
901 )
902 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
903 requires_casting = connection.features.requires_casted_case_in_updates
904 batches = (objs[i : i + batch_size] for i in range(0, len(objs), batch_size))
905 updates = []
906 for batch_objs in batches:
907 update_kwargs = {}
908 for field in fields:
909 when_statements = []
910 for obj in batch_objs:
911 attr = getattr(obj, field.attname)
912 if not hasattr(attr, "resolve_expression"):
913 attr = Value(attr, output_field=field)
914 when_statements.append(When(pk=obj.pk, then=attr))
915 case_statement = Case(*when_statements, output_field=field)
916 if requires_casting:
917 case_statement = Cast(case_statement, output_field=field)
918 update_kwargs[field.attname] = case_statement
919 updates.append(([obj.pk for obj in batch_objs], update_kwargs))
920 rows_updated = 0
921 queryset = self.using(self.db)
922 with transaction.atomic(using=self.db, savepoint=False):
923 for pks, update_kwargs in updates:
924 rows_updated += queryset.filter(pk__in=pks).update(**update_kwargs)
925 return rows_updated
926
927 bulk_update.alters_data = True
928
929 async def abulk_update(self, objs, fields, batch_size=None):
930 return await sync_to_async(self.bulk_update)(
931 objs=objs,
932 fields=fields,
933 batch_size=batch_size,
934 )
935
936 abulk_update.alters_data = True
937
938 def get_or_create(self, defaults=None, **kwargs):
939 """
940 Look up an object with the given kwargs, creating one if necessary.
941 Return a tuple of (object, created), where created is a boolean
942 specifying whether an object was created.
943 """
944 # The get() needs to be targeted at the write database in order
945 # to avoid potential transaction consistency problems.
946 self._for_write = True
947 try:
948 return self.get(**kwargs), False
949 except self.model.DoesNotExist:
950 params = self._extract_model_params(defaults, **kwargs)
951 # Try to create an object using passed params.
952 try:
953 with transaction.atomic(using=self.db):
954 params = dict(resolve_callables(params))
955 return self.create(**params), True
956 except IntegrityError:
957 try:
958 return self.get(**kwargs), False
959 except self.model.DoesNotExist:
960 pass
961 raise
962
963 get_or_create.alters_data = True
964
965 async def aget_or_create(self, defaults=None, **kwargs):
966 return await sync_to_async(self.get_or_create)(
967 defaults=defaults,
968 **kwargs,
969 )
970
971 aget_or_create.alters_data = True
972
973 def update_or_create(self, defaults=None, create_defaults=None, **kwargs):
974 """
975 Look up an object with the given kwargs, updating one with defaults
976 if it exists, otherwise create a new one. Optionally, an object can
977 be created with different values than defaults by using
978 create_defaults.
979 Return a tuple (object, created), where created is a boolean
980 specifying whether an object was created.
981 """
982 update_defaults = defaults or {}
983 if create_defaults is None:
984 create_defaults = update_defaults
985
986 self._for_write = True
987 with transaction.atomic(using=self.db):
988 # Lock the row so that a concurrent update is blocked until
989 # update_or_create() has performed its save.
990 obj, created = self.select_for_update().get_or_create(
991 create_defaults, **kwargs
992 )
993 if created:
994 return obj, created
995 for k, v in resolve_callables(update_defaults):
996 setattr(obj, k, v)
997
998 update_fields = set(update_defaults)
999 concrete_field_names = self.model._meta._non_pk_concrete_field_names
1000 # update_fields does not support non-concrete fields.
1001 if concrete_field_names.issuperset(update_fields):
1002 # Add fields which are set on pre_save(), e.g. auto_now fields.
1003 # This is to maintain backward compatibility as these fields
1004 # are not updated unless explicitly specified in the
1005 # update_fields list.
1006 pk_fields = self.model._meta.pk_fields
1007 for field in self.model._meta.local_concrete_fields:
1008 if not (
1009 field in pk_fields or field.__class__.pre_save is Field.pre_save
1010 ):
1011 update_fields.add(field.name)
1012 if field.name != field.attname:
1013 update_fields.add(field.attname)
1014 obj.save(using=self.db, update_fields=update_fields)
1015 else:
1016 obj.save(using=self.db)
1017 return obj, False
1018
1019 update_or_create.alters_data = True
1020
1021 async def aupdate_or_create(self, defaults=None, create_defaults=None, **kwargs):
1022 return await sync_to_async(self.update_or_create)(
1023 defaults=defaults,
1024 create_defaults=create_defaults,
1025 **kwargs,
1026 )
1027
1028 aupdate_or_create.alters_data = True
1029
1030 def _extract_model_params(self, defaults, **kwargs):
1031 """
1032 Prepare `params` for creating a model instance based on the given
1033 kwargs; for use by get_or_create().
1034 """
1035 defaults = defaults or {}
1036 params = {k: v for k, v in kwargs.items() if LOOKUP_SEP not in k}
1037 params.update(defaults)
1038 property_names = self.model._meta._property_names
1039 invalid_params = []
1040 for param in params:
1041 try:
1042 self.model._meta.get_field(param)
1043 except exceptions.FieldDoesNotExist:
1044 # It's okay to use a model's property if it has a setter.
1045 if not (param in property_names and getattr(self.model, param).fset):
1046 invalid_params.append(param)
1047 if invalid_params:
1048 raise exceptions.FieldError(
1049 "Invalid field name(s) for model %s: '%s'."
1050 % (
1051 self.model._meta.object_name,
1052 "', '".join(sorted(invalid_params)),
1053 )
1054 )
1055 return params
1056
1057 def _earliest(self, *fields):
1058 """
1059 Return the earliest object according to fields (if given) or by the
1060 model's Meta.get_latest_by.
1061 """
1062 if fields:
1063 order_by = fields
1064 else:
1065 order_by = getattr(self.model._meta, "get_latest_by")
1066 if order_by and not isinstance(order_by, (tuple, list)):
1067 order_by = (order_by,)
1068 if order_by is None:
1069 raise ValueError(
1070 "earliest() and latest() require either fields as positional "
1071 "arguments or 'get_latest_by' in the model's Meta."
1072 )
1073 obj = self._chain()
1074 obj.query.set_limits(high=1)
1075 obj.query.clear_ordering(force=True)
1076 obj.query.add_ordering(*order_by)
1077 return obj.get()
1078
1079 def earliest(self, *fields):
1080 if self.query.is_sliced:
1081 raise TypeError("Cannot change a query once a slice has been taken.")
1082 return self._earliest(*fields)
1083
1084 async def aearliest(self, *fields):
1085 return await sync_to_async(self.earliest)(*fields)
1086
1087 def latest(self, *fields):
1088 """
1089 Return the latest object according to fields (if given) or by the
1090 model's Meta.get_latest_by.
1091 """
1092 if self.query.is_sliced:
1093 raise TypeError("Cannot change a query once a slice has been taken.")
1094 return self.reverse()._earliest(*fields)
1095
1096 async def alatest(self, *fields):
1097 return await sync_to_async(self.latest)(*fields)
1098
1099 def first(self):
1100 """Return the first object of a query or None if no match is found."""
1101 if self.ordered:
1102 queryset = self
1103 else:
1104 self._check_ordering_first_last_queryset_aggregation(method="first")
1105 queryset = self.order_by("pk")
1106 for obj in queryset[:1]:
1107 return obj
1108
1109 async def afirst(self):
1110 return await sync_to_async(self.first)()
1111
1112 def last(self):
1113 """Return the last object of a query or None if no match is found."""
1114 if self.ordered:
1115 queryset = self.reverse()
1116 else:
1117 self._check_ordering_first_last_queryset_aggregation(method="last")
1118 queryset = self.order_by("-pk")
1119 for obj in queryset[:1]:
1120 return obj
1121
1122 async def alast(self):
1123 return await sync_to_async(self.last)()
1124
1125 def in_bulk(self, id_list=None, *, field_name="pk"):
1126 """
1127 Return a dictionary mapping each of the given IDs to the object with
1128 that ID. If `id_list` isn't provided, evaluate the entire QuerySet.
1129 """
1130 if self.query.is_sliced:
1131 raise TypeError("Cannot use 'limit' or 'offset' with in_bulk().")
1132 if not issubclass(self._iterable_class, ModelIterable):
1133 raise TypeError("in_bulk() cannot be used with values() or values_list().")
1134 opts = self.model._meta
1135 unique_fields = [
1136 constraint.fields[0]
1137 for constraint in opts.total_unique_constraints
1138 if len(constraint.fields) == 1
1139 ]
1140 if (
1141 field_name != "pk"
1142 and not opts.get_field(field_name).unique
1143 and field_name not in unique_fields
1144 and self.query.distinct_fields != (field_name,)
1145 ):
1146 raise ValueError(
1147 "in_bulk()'s field_name must be a unique field but %r isn't."
1148 % field_name
1149 )
1150 if id_list is not None:
1151 if not id_list:
1152 return {}
1153 filter_key = "{}__in".format(field_name)
1154 max_params = connections[self.db].features.max_query_params or 0
1155 num_fields = len(opts.pk_fields) if field_name == "pk" else 1
1156 batch_size = max_params // num_fields
1157 id_list = tuple(id_list)
1158 # If the database has a limit on the number of query parameters
1159 # (e.g. SQLite), retrieve objects in batches if necessary.
1160 if batch_size and batch_size < len(id_list):
1161 qs = ()
1162 for offset in range(0, len(id_list), batch_size):
1163 batch = id_list[offset : offset + batch_size]
1164 qs += tuple(self.filter(**{filter_key: batch}))
1165 else:
1166 qs = self.filter(**{filter_key: id_list})
1167 else:
1168 qs = self._chain()
1169 return {getattr(obj, field_name): obj for obj in qs}
1170
1171 async def ain_bulk(self, id_list=None, *, field_name="pk"):
1172 return await sync_to_async(self.in_bulk)(
1173 id_list=id_list,
1174 field_name=field_name,
1175 )
1176
1177 def delete(self):
1178 """Delete the records in the current QuerySet."""
1179 self._not_support_combined_queries("delete")
1180 if self.query.is_sliced:
1181 raise TypeError("Cannot use 'limit' or 'offset' with delete().")
1182 if self.query.distinct_fields:
1183 raise TypeError("Cannot call delete() after .distinct(*fields).")
1184 if self._fields is not None:
1185 raise TypeError("Cannot call delete() after .values() or .values_list()")
1186
1187 del_query = self._chain()
1188
1189 # The delete is actually 2 queries - one to find related objects,
1190 # and one to delete. Make sure that the discovery of related
1191 # objects is performed on the same database as the deletion.
1192 del_query._for_write = True
1193
1194 # Disable non-supported fields.
1195 del_query.query.select_for_update = False
1196 del_query.query.select_related = False
1197 del_query.query.clear_ordering(force=True)
1198
1199 collector = Collector(using=del_query.db, origin=self)
1200 collector.collect(del_query)
1201 num_deleted, num_deleted_per_model = collector.delete()
1202
1203 # Clear the result cache, in case this QuerySet gets reused.
1204 self._result_cache = None
1205 return num_deleted, num_deleted_per_model
1206
1207 delete.alters_data = True
1208 delete.queryset_only = True
1209
1210 async def adelete(self):
1211 return await sync_to_async(self.delete)()
1212
1213 adelete.alters_data = True
1214 adelete.queryset_only = True
1215
1216 def _raw_delete(self, using):
1217 """
1218 Delete objects found from the given queryset in single direct SQL
1219 query. No signals are sent and there is no protection for cascades.
1220 """
1221 query = self.query.clone()
1222 query.__class__ = sql.DeleteQuery
1223 return query.get_compiler(using).execute_sql(ROW_COUNT)
1224
1225 _raw_delete.alters_data = True
1226
1227 def update(self, **kwargs):
1228 """
1229 Update all elements in the current QuerySet, setting all the given
1230 fields to the appropriate values.
1231 """
1232 self._not_support_combined_queries("update")
1233 if self.query.is_sliced:
1234 raise TypeError("Cannot update a query once a slice has been taken.")
1235 self._for_write = True
1236 query = self.query.chain(sql.UpdateQuery)
1237 query.add_update_values(kwargs)
1238
1239 # Inline annotations in order_by(), if possible.
1240 new_order_by = []
1241 for col in query.order_by:
1242 alias = col
1243 descending = False
1244 if isinstance(alias, str) and alias.startswith("-"):
1245 alias = alias.removeprefix("-")
1246 descending = True
1247 if annotation := query.annotations.get(alias):
1248 if getattr(annotation, "contains_aggregate", False):
1249 raise exceptions.FieldError(
1250 f"Cannot update when ordering by an aggregate: {annotation}"
1251 )
1252 if descending:
1253 annotation = annotation.desc()
1254 new_order_by.append(annotation)
1255 else:
1256 new_order_by.append(col)
1257 query.order_by = tuple(new_order_by)
1258
1259 # Clear SELECT clause as all annotation references were inlined by
1260 # add_update_values() already.
1261 query.clear_select_clause()
1262 with transaction.mark_for_rollback_on_error(using=self.db):
1263 rows = query.get_compiler(self.db).execute_sql(ROW_COUNT)
1264 self._result_cache = None
1265 return rows
1266
1267 update.alters_data = True
1268
1269 async def aupdate(self, **kwargs):
1270 return await sync_to_async(self.update)(**kwargs)
1271
1272 aupdate.alters_data = True
1273
1274 def _update(self, values):
1275 """
1276 A version of update() that accepts field objects instead of field names.
1277 Used primarily for model saving and not intended for use by general
1278 code (it requires too much poking around at model internals to be
1279 useful at that level).
1280 """
1281 if self.query.is_sliced:
1282 raise TypeError("Cannot update a query once a slice has been taken.")
1283 query = self.query.chain(sql.UpdateQuery)
1284 query.add_update_fields(values)
1285 # Clear any annotations so that they won't be present in subqueries.
1286 query.annotations = {}
1287 self._result_cache = None
1288 return query.get_compiler(self.db).execute_sql(ROW_COUNT)
1289
1290 _update.alters_data = True
1291 _update.queryset_only = False
1292
1293 def exists(self):
1294 """
1295 Return True if the QuerySet would have any results, False otherwise.
1296 """
1297 if self._result_cache is None:
1298 return self.query.has_results(using=self.db)
1299 return bool(self._result_cache)
1300
1301 async def aexists(self):
1302 return await sync_to_async(self.exists)()
1303
1304 def contains(self, obj):
1305 """
1306 Return True if the QuerySet contains the provided obj,
1307 False otherwise.
1308 """
1309 self._not_support_combined_queries("contains")
1310 if self._fields is not None:
1311 raise TypeError(
1312 "Cannot call QuerySet.contains() after .values() or .values_list()."
1313 )
1314 try:
1315 if obj._meta.concrete_model != self.model._meta.concrete_model:
1316 return False
1317 except AttributeError:
1318 raise TypeError("'obj' must be a model instance.")
1319 if not obj._is_pk_set():
1320 raise ValueError("QuerySet.contains() cannot be used on unsaved objects.")
1321 if self._result_cache is not None:
1322 return obj in self._result_cache
1323 return self.filter(pk=obj.pk).exists()
1324
1325 async def acontains(self, obj):
1326 return await sync_to_async(self.contains)(obj=obj)
1327
1328 def _prefetch_related_objects(self):
1329 # This method can only be called once the result cache has been filled.
1330 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
1331 self._prefetch_done = True
1332
1333 def explain(self, *, format=None, **options):
1334 """
1335 Runs an EXPLAIN on the SQL query this QuerySet would perform, and
1336 returns the results.
1337 """
1338 return self.query.explain(using=self.db, format=format, **options)
1339
1340 async def aexplain(self, *, format=None, **options):
1341 return await sync_to_async(self.explain)(format=format, **options)
1342
1343 ##################################################
1344 # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
1345 ##################################################
1346
1347 def raw(self, raw_query, params=(), translations=None, using=None):
1348 if using is None:
1349 using = self.db
1350 qs = RawQuerySet(
1351 raw_query,
1352 model=self.model,
1353 params=params,
1354 translations=translations,
1355 using=using,
1356 )
1357 qs._prefetch_related_lookups = self._prefetch_related_lookups[:]
1358 return qs
1359
1360 def _values(self, *fields, **expressions):
1361 clone = self._chain()
1362 if expressions:
1363 clone = clone.annotate(**expressions)
1364 clone._fields = fields
1365 clone.query.set_values(fields)
1366 return clone
1367
1368 def values(self, *fields, **expressions):
1369 fields += tuple(expressions)
1370 clone = self._values(*fields, **expressions)
1371 clone._iterable_class = ValuesIterable
1372 return clone
1373
1374 def values_list(self, *fields, flat=False, named=False):
1375 if flat and named:
1376 raise TypeError("'flat' and 'named' can't be used together.")
1377 if flat and len(fields) > 1:
1378 raise TypeError(
1379 "'flat' is not valid when values_list is called with more than one "
1380 "field."
1381 )
1382
1383 field_names = {f: False for f in fields if not hasattr(f, "resolve_expression")}
1384 _fields = []
1385 expressions = {}
1386 counter = 1
1387 for field in fields:
1388 field_name = field
1389 expression = None
1390 if hasattr(field, "resolve_expression"):
1391 field_name = getattr(
1392 field, "default_alias", field.__class__.__name__.lower()
1393 )
1394 expression = field
1395 # For backward compatibility reasons expressions are always
1396 # prefixed with the counter even if their default alias doesn't
1397 # collide with field names. Changing this logic could break
1398 # some usage of named=True.
1399 seen = True
1400 elif seen := field_names[field_name]:
1401 expression = F(field_name)
1402 if seen:
1403 field_name_prefix = field_name
1404 while (field_name := f"{field_name_prefix}{counter}") in field_names:
1405 counter += 1
1406 if expression is not None:
1407 expressions[field_name] = expression
1408 field_names[field_name] = True
1409 _fields.append(field_name)
1410
1411 clone = self._values(*_fields, **expressions)
1412 clone._iterable_class = (
1413 NamedValuesListIterable
1414 if named
1415 else FlatValuesListIterable if flat else ValuesListIterable
1416 )
1417 return clone
1418
1419 def dates(self, field_name, kind, order="ASC"):
1420 """
1421 Return a list of date objects representing all available dates for
1422 the given field_name, scoped to 'kind'.
1423 """
1424 if kind not in ("year", "month", "week", "day"):
1425 raise ValueError("'kind' must be one of 'year', 'month', 'week', or 'day'.")
1426 if order not in ("ASC", "DESC"):
1427 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1428 return (
1429 self.annotate(
1430 datefield=Trunc(field_name, kind, output_field=DateField()),
1431 plain_field=F(field_name),
1432 )
1433 .values_list("datefield", flat=True)
1434 .distinct()
1435 .filter(plain_field__isnull=False)
1436 .order_by(("-" if order == "DESC" else "") + "datefield")
1437 )
1438
1439 def datetimes(self, field_name, kind, order="ASC", tzinfo=None):
1440 """
1441 Return a list of datetime objects representing all available
1442 datetimes for the given field_name, scoped to 'kind'.
1443 """
1444 if kind not in ("year", "month", "week", "day", "hour", "minute", "second"):
1445 raise ValueError(
1446 "'kind' must be one of 'year', 'month', 'week', 'day', "
1447 "'hour', 'minute', or 'second'."
1448 )
1449 if order not in ("ASC", "DESC"):
1450 raise ValueError("'order' must be either 'ASC' or 'DESC'.")
1451 if settings.USE_TZ:
1452 if tzinfo is None:
1453 tzinfo = timezone.get_current_timezone()
1454 else:
1455 tzinfo = None
1456 return (
1457 self.annotate(
1458 datetimefield=Trunc(
1459 field_name,
1460 kind,
1461 output_field=DateTimeField(),
1462 tzinfo=tzinfo,
1463 ),
1464 plain_field=F(field_name),
1465 )
1466 .values_list("datetimefield", flat=True)
1467 .distinct()
1468 .filter(plain_field__isnull=False)
1469 .order_by(("-" if order == "DESC" else "") + "datetimefield")
1470 )
1471
1472 def none(self):
1473 """Return an empty QuerySet."""
1474 clone = self._chain()
1475 clone.query.set_empty()
1476 return clone
1477
1478 ##################################################################
1479 # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET #
1480 ##################################################################
1481
1482 def all(self):
1483 """
1484 Return a new QuerySet that is a copy of the current one. This allows a
1485 QuerySet to proxy for a model manager in some cases.
1486 """
1487 return self._chain()
1488
1489 def filter(self, *args, **kwargs):
1490 """
1491 Return a new QuerySet instance with the args ANDed to the existing
1492 set.
1493 """
1494 self._not_support_combined_queries("filter")
1495 return self._filter_or_exclude(False, args, kwargs)
1496
1497 def exclude(self, *args, **kwargs):
1498 """
1499 Return a new QuerySet instance with NOT (args) ANDed to the existing
1500 set.
1501 """
1502 self._not_support_combined_queries("exclude")
1503 return self._filter_or_exclude(True, args, kwargs)
1504
1505 def _filter_or_exclude(self, negate, args, kwargs):
1506 if (args or kwargs) and self.query.is_sliced:
1507 raise TypeError("Cannot filter a query once a slice has been taken.")
1508 clone = self._chain()
1509 if self._defer_next_filter:
1510 self._defer_next_filter = False
1511 clone._deferred_filter = negate, args, kwargs
1512 else:
1513 clone._filter_or_exclude_inplace(negate, args, kwargs)
1514 return clone
1515
1516 def _filter_or_exclude_inplace(self, negate, args, kwargs):
1517 if invalid_kwargs := PROHIBITED_FILTER_KWARGS.intersection(kwargs):
1518 invalid_kwargs_str = ", ".join(f"'{k}'" for k in sorted(invalid_kwargs))
1519 raise TypeError(f"The following kwargs are invalid: {invalid_kwargs_str}")
1520 if negate:
1521 self._query.add_q(~Q(*args, **kwargs))
1522 else:
1523 self._query.add_q(Q(*args, **kwargs))
1524
1525 def complex_filter(self, filter_obj):
1526 """
1527 Return a new QuerySet instance with filter_obj added to the filters.
1528
1529 filter_obj can be a Q object or a dictionary of keyword lookup
1530 arguments.
1531
1532 This exists to support framework features such as 'limit_choices_to',
1533 and usually it will be more natural to use other methods.
1534 """
1535 if isinstance(filter_obj, Q):
1536 clone = self._chain()
1537 clone.query.add_q(filter_obj)
1538 return clone
1539 else:
1540 return self._filter_or_exclude(False, args=(), kwargs=filter_obj)
1541
1542 def _combinator_query(self, combinator, *other_qs, all=False):
1543 # Clone the query to inherit the select list and everything
1544 clone = self._chain()
1545 # Clear limits and ordering so they can be reapplied
1546 clone.query.clear_ordering(force=True)
1547 clone.query.clear_limits()
1548 clone.query.combined_queries = (self.query,) + tuple(
1549 qs.query for qs in other_qs
1550 )
1551 clone.query.combinator = combinator
1552 clone.query.combinator_all = all
1553 return clone
1554
1555 def union(self, *other_qs, all=False):
1556 # If the query is an EmptyQuerySet, combine all nonempty querysets.
1557 if isinstance(self, EmptyQuerySet):
1558 qs = [q for q in other_qs if not isinstance(q, EmptyQuerySet)]
1559 if not qs:
1560 return self
1561 if len(qs) == 1:
1562 return qs[0]
1563 return qs[0]._combinator_query("union", *qs[1:], all=all)
1564 elif not other_qs:
1565 return self
1566 return self._combinator_query("union", *other_qs, all=all)
1567
1568 def intersection(self, *other_qs):
1569 # If any query is an EmptyQuerySet, return it.
1570 if isinstance(self, EmptyQuerySet):
1571 return self
1572 for other in other_qs:
1573 if isinstance(other, EmptyQuerySet):
1574 return other
1575 return self._combinator_query("intersection", *other_qs)
1576
1577 def difference(self, *other_qs):
1578 # If the query is an EmptyQuerySet, return it.
1579 if isinstance(self, EmptyQuerySet):
1580 return self
1581 return self._combinator_query("difference", *other_qs)
1582
1583 def select_for_update(self, nowait=False, skip_locked=False, of=(), no_key=False):
1584 """
1585 Return a new QuerySet instance that will select objects with a
1586 FOR UPDATE lock.
1587 """
1588 if nowait and skip_locked:
1589 raise ValueError("The nowait option cannot be used with skip_locked.")
1590 obj = self._chain()
1591 obj._for_write = True
1592 obj.query.select_for_update = True
1593 obj.query.select_for_update_nowait = nowait
1594 obj.query.select_for_update_skip_locked = skip_locked
1595 obj.query.select_for_update_of = of
1596 obj.query.select_for_no_key_update = no_key
1597 return obj
1598
1599 def select_related(self, *fields):
1600 """
1601 Return a new QuerySet instance that will select related objects.
1602
1603 If fields are specified, they must be ForeignKey fields and only those
1604 related objects are included in the selection.
1605
1606 If select_related(None) is called, clear the list.
1607 """
1608 self._not_support_combined_queries("select_related")
1609 if self._fields is not None:
1610 raise TypeError(
1611 "Cannot call select_related() after .values() or .values_list()"
1612 )
1613
1614 obj = self._chain()
1615 if fields == (None,):
1616 obj.query.select_related = False
1617 elif fields:
1618 obj.query.add_select_related(fields)
1619 else:
1620 obj.query.select_related = True
1621 return obj
1622
1623 def prefetch_related(self, *lookups):
1624 """
1625 Return a new QuerySet instance that will prefetch the specified
1626 Many-To-One and Many-To-Many related objects when the QuerySet is
1627 evaluated.
1628
1629 When prefetch_related() is called more than once, append to the list of
1630 prefetch lookups. If prefetch_related(None) is called, clear the list.
1631 """
1632 self._not_support_combined_queries("prefetch_related")
1633 clone = self._chain()
1634 if lookups == (None,):
1635 clone._prefetch_related_lookups = ()
1636 else:
1637 for lookup in lookups:
1638 if isinstance(lookup, Prefetch):
1639 lookup = lookup.prefetch_to
1640 lookup = lookup.split(LOOKUP_SEP, 1)[0]
1641 if lookup in self.query._filtered_relations:
1642 raise ValueError(
1643 "prefetch_related() is not supported with FilteredRelation."
1644 )
1645 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
1646 return clone
1647
1648 def annotate(self, *args, **kwargs):
1649 """
1650 Return a query set in which the returned objects have been annotated
1651 with extra data or aggregations.
1652 """
1653 self._not_support_combined_queries("annotate")
1654 return self._annotate(args, kwargs, select=True)
1655
1656 def alias(self, *args, **kwargs):
1657 """
1658 Return a query set with added aliases for extra data or aggregations.
1659 """
1660 self._not_support_combined_queries("alias")
1661 return self._annotate(args, kwargs, select=False)
1662
1663 def _annotate(self, args, kwargs, select=True):
1664 self._validate_values_are_expressions(
1665 args + tuple(kwargs.values()), method_name="annotate"
1666 )
1667 annotations = {}
1668 for arg in args:
1669 # The default_alias property raises TypeError if default_alias
1670 # can't be set automatically or AttributeError if it isn't an
1671 # attribute.
1672 try:
1673 if arg.default_alias in kwargs:
1674 raise ValueError(
1675 "The named annotation '%s' conflicts with the "
1676 "default name for another annotation." % arg.default_alias
1677 )
1678 except (TypeError, AttributeError):
1679 raise TypeError("Complex annotations require an alias")
1680 annotations[arg.default_alias] = arg
1681 annotations.update(kwargs)
1682
1683 clone = self._chain()
1684 names = self._fields
1685 if names is None:
1686 names = set(
1687 chain.from_iterable(
1688 (
1689 (field.name, field.attname)
1690 if hasattr(field, "attname")
1691 else (field.name,)
1692 )
1693 for field in self.model._meta.get_fields()
1694 )
1695 )
1696
1697 for alias, annotation in annotations.items():
1698 if alias in names:
1699 raise ValueError(
1700 "The annotation '%s' conflicts with a field on "
1701 "the model." % alias
1702 )
1703 if isinstance(annotation, FilteredRelation):
1704 clone.query.add_filtered_relation(annotation, alias)
1705 else:
1706 clone.query.add_annotation(
1707 annotation,
1708 alias,
1709 select=select,
1710 )
1711 for alias, annotation in clone.query.annotations.items():
1712 if alias in annotations and annotation.contains_aggregate:
1713 if clone._fields is None:
1714 clone.query.group_by = True
1715 else:
1716 clone.query.set_group_by()
1717 break
1718
1719 return clone
1720
1721 def order_by(self, *field_names):
1722 """Return a new QuerySet instance with the ordering changed."""
1723 if self.query.is_sliced:
1724 raise TypeError("Cannot reorder a query once a slice has been taken.")
1725 obj = self._chain()
1726 obj.query.clear_ordering(force=True, clear_default=False)
1727 obj.query.add_ordering(*field_names)
1728 return obj
1729
1730 def distinct(self, *field_names):
1731 """
1732 Return a new QuerySet instance that will select only distinct results.
1733 """
1734 self._not_support_combined_queries("distinct")
1735 if self.query.is_sliced:
1736 raise TypeError(
1737 "Cannot create distinct fields once a slice has been taken."
1738 )
1739 obj = self._chain()
1740 obj.query.add_distinct_fields(*field_names)
1741 return obj
1742
1743 def extra(
1744 self,
1745 select=None,
1746 where=None,
1747 params=None,
1748 tables=None,
1749 order_by=None,
1750 select_params=None,
1751 ):
1752 """Add extra SQL fragments to the query."""
1753 self._not_support_combined_queries("extra")
1754 if self.query.is_sliced:
1755 raise TypeError("Cannot change a query once a slice has been taken.")
1756 clone = self._chain()
1757 clone.query.add_extra(select, select_params, where, params, tables, order_by)
1758 return clone
1759
1760 def reverse(self):
1761 """Reverse the ordering of the QuerySet."""
1762 if self.query.is_sliced:
1763 raise TypeError("Cannot reverse a query once a slice has been taken.")
1764 clone = self._chain()
1765 clone.query.standard_ordering = not clone.query.standard_ordering
1766 return clone
1767
1768 def defer(self, *fields):
1769 """
1770 Defer the loading of data for certain fields until they are accessed.
1771 Add the set of deferred fields to any existing set of deferred fields.
1772 The only exception to this is if None is passed in as the only
1773 parameter, in which case remove all deferrals.
1774 """
1775 self._not_support_combined_queries("defer")
1776 if self._fields is not None:
1777 raise TypeError("Cannot call defer() after .values() or .values_list()")
1778 clone = self._chain()
1779 if fields == (None,):
1780 clone.query.clear_deferred_loading()
1781 else:
1782 clone.query.add_deferred_loading(fields)
1783 return clone
1784
1785 def only(self, *fields):
1786 """
1787 Essentially, the opposite of defer(). Only the fields passed into this
1788 method and that are not already specified as deferred are loaded
1789 immediately when the queryset is evaluated.
1790 """
1791 self._not_support_combined_queries("only")
1792 if self._fields is not None:
1793 raise TypeError("Cannot call only() after .values() or .values_list()")
1794 if fields == (None,):
1795 # Can only pass None to defer(), not only(), as the rest option.
1796 # That won't stop people trying to do this, so let's be explicit.
1797 raise TypeError("Cannot pass None as an argument to only().")
1798 for field in fields:
1799 field = field.split(LOOKUP_SEP, 1)[0]
1800 if field in self.query._filtered_relations:
1801 raise ValueError("only() is not supported with FilteredRelation.")
1802 clone = self._chain()
1803 clone.query.add_immediate_loading(fields)
1804 return clone
1805
1806 def using(self, alias):
1807 """Select which database this QuerySet should execute against."""
1808 clone = self._chain()
1809 clone._db = alias
1810 return clone
1811
1812 ###################################
1813 # PUBLIC INTROSPECTION ATTRIBUTES #
1814 ###################################
1815
1816 @property
1817 def ordered(self):
1818 """
1819 Return True if the QuerySet is ordered -- i.e. has an order_by()
1820 clause or a default ordering on the model (or is empty).
1821 """
1822 if isinstance(self, EmptyQuerySet):
1823 return True
1824 if self.query.extra_order_by or self.query.order_by:
1825 return True
1826 elif (
1827 self.query.default_ordering
1828 and self.query.get_meta().ordering
1829 and
1830 # A default ordering doesn't affect GROUP BY queries.
1831 not self.query.group_by
1832 ):
1833 return True
1834 else:
1835 return False
1836
1837 @property
1838 def db(self):
1839 """Return the database used if this query is executed now."""
1840 if self._for_write:
1841 return self._db or router.db_for_write(self.model, **self._hints)
1842 return self._db or router.db_for_read(self.model, **self._hints)
1843
1844 ###################
1845 # PRIVATE METHODS #
1846 ###################
1847
1848 def _insert(
1849 self,
1850 objs,
1851 fields,
1852 returning_fields=None,
1853 raw=False,
1854 using=None,
1855 on_conflict=None,
1856 update_fields=None,
1857 unique_fields=None,
1858 ):
1859 """
1860 Insert a new record for the given model. This provides an interface to
1861 the InsertQuery class and is how Model.save() is implemented.
1862 """
1863 self._for_write = True
1864 if using is None:
1865 using = self.db
1866 query = sql.InsertQuery(
1867 self.model,
1868 on_conflict=on_conflict,
1869 update_fields=update_fields,
1870 unique_fields=unique_fields,
1871 )
1872 query.insert_values(fields, objs, raw=raw)
1873 return query.get_compiler(using=using).execute_sql(returning_fields)
1874
1875 _insert.alters_data = True
1876 _insert.queryset_only = False
1877
1878 def _batched_insert(
1879 self,
1880 objs,
1881 fields,
1882 batch_size,
1883 on_conflict=None,
1884 update_fields=None,
1885 unique_fields=None,
1886 ):
1887 """
1888 Helper method for bulk_create() to insert objs one batch at a time.
1889 """
1890 connection = connections[self.db]
1891 ops = connection.ops
1892 max_batch_size = max(ops.bulk_batch_size(fields, objs), 1)
1893 batch_size = min(batch_size, max_batch_size) if batch_size else max_batch_size
1894 inserted_rows = []
1895 bulk_return = connection.features.can_return_rows_from_bulk_insert
1896 for item in [objs[i : i + batch_size] for i in range(0, len(objs), batch_size)]:
1897 if bulk_return and (
1898 on_conflict is None or on_conflict == OnConflict.UPDATE
1899 ):
1900 inserted_rows.extend(
1901 self._insert(
1902 item,
1903 fields=fields,
1904 using=self.db,
1905 on_conflict=on_conflict,
1906 update_fields=update_fields,
1907 unique_fields=unique_fields,
1908 returning_fields=self.model._meta.db_returning_fields,
1909 )
1910 )
1911 else:
1912 self._insert(
1913 item,
1914 fields=fields,
1915 using=self.db,
1916 on_conflict=on_conflict,
1917 update_fields=update_fields,
1918 unique_fields=unique_fields,
1919 )
1920 return inserted_rows
1921
1922 def _chain(self):
1923 """
1924 Return a copy of the current QuerySet that's ready for another
1925 operation.
1926 """
1927 obj = self._clone()
1928 if obj._sticky_filter:
1929 obj.query.filter_is_sticky = True
1930 obj._sticky_filter = False
1931 return obj
1932
1933 def _clone(self):
1934 """
1935 Return a copy of the current QuerySet. A lightweight alternative
1936 to deepcopy().
1937 """
1938 c = self.__class__(
1939 model=self.model,
1940 query=self.query.chain(),
1941 using=self._db,
1942 hints=self._hints,
1943 )
1944 c._sticky_filter = self._sticky_filter
1945 c._for_write = self._for_write
1946 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
1947 c._known_related_objects = self._known_related_objects
1948 c._iterable_class = self._iterable_class
1949 c._fields = self._fields
1950 return c
1951
1952 def _fetch_all(self):
1953 if self._result_cache is None:
1954 self._result_cache = list(self._iterable_class(self))
1955 if self._prefetch_related_lookups and not self._prefetch_done:
1956 self._prefetch_related_objects()
1957
1958 def _next_is_sticky(self):
1959 """
1960 Indicate that the next filter call and the one following that should
1961 be treated as a single filter. This is only important when it comes to
1962 determining when to reuse tables for many-to-many filters. Required so
1963 that we can filter naturally on the results of related managers.
1964
1965 This doesn't return a clone of the current QuerySet (it returns
1966 "self"). The method is only used internally and should be immediately
1967 followed by a filter() that does create a clone.
1968 """
1969 self._sticky_filter = True
1970 return self
1971
1972 def _merge_sanity_check(self, other):
1973 """Check that two QuerySet classes may be merged."""
1974 if self._fields is not None and (
1975 set(self.query.values_select) != set(other.query.values_select)
1976 or set(self.query.extra_select) != set(other.query.extra_select)
1977 or set(self.query.annotation_select) != set(other.query.annotation_select)
1978 ):
1979 raise TypeError(
1980 "Merging '%s' classes must involve the same values in each case."
1981 % self.__class__.__name__
1982 )
1983
1984 def _merge_known_related_objects(self, other):
1985 """
1986 Keep track of all known related objects from either QuerySet instance.
1987 """
1988 for field, objects in other._known_related_objects.items():
1989 self._known_related_objects.setdefault(field, {}).update(objects)
1990
1991 def resolve_expression(self, *args, **kwargs):
1992 query = self.query.resolve_expression(*args, **kwargs)
1993 query._db = self._db
1994 return query
1995
1996 resolve_expression.queryset_only = True
1997
1998 def _add_hints(self, **hints):
1999 """
2000 Update hinting information for use by routers. Add new key/values or
2001 overwrite existing key/values.
2002 """
2003 self._hints.update(hints)
2004
2005 def _has_filters(self):
2006 """
2007 Check if this QuerySet has any filtering going on. This isn't
2008 equivalent with checking if all objects are present in results, for
2009 example, qs[1:]._has_filters() -> False.
2010 """
2011 return self.query.has_filters()
2012
2013 @staticmethod
2014 def _validate_values_are_expressions(values, method_name):
2015 invalid_args = sorted(
2016 str(arg) for arg in values if not hasattr(arg, "resolve_expression")
2017 )
2018 if invalid_args:
2019 raise TypeError(
2020 "QuerySet.%s() received non-expression(s): %s."
2021 % (
2022 method_name,
2023 ", ".join(invalid_args),
2024 )
2025 )
2026
2027 def _not_support_combined_queries(self, operation_name):
2028 if self.query.combinator:
2029 raise NotSupportedError(
2030 "Calling QuerySet.%s() after %s() is not supported."
2031 % (operation_name, self.query.combinator)
2032 )
2033
2034 def _check_operator_queryset(self, other, operator_):
2035 if self.query.combinator or other.query.combinator:
2036 raise TypeError(f"Cannot use {operator_} operator with combined queryset.")
2037
2038 def _check_ordering_first_last_queryset_aggregation(self, method):
2039 if (
2040 isinstance(self.query.group_by, tuple)
2041 # Raise if the pk fields are not in the group_by.
2042 and self.model._meta.pk
2043 not in {col.output_field for col in self.query.group_by}
2044 and set(self.model._meta.pk_fields).difference(
2045 {col.target for col in self.query.group_by}
2046 )
2047 ):
2048 raise TypeError(
2049 f"Cannot use QuerySet.{method}() on an unordered queryset performing "
2050 f"aggregation. Add an ordering with order_by()."
2051 )
2052
2053
2054 class InstanceCheckMeta(type):
2055 def __instancecheck__(self, instance):
2056 return isinstance(instance, QuerySet) and instance.query.is_empty()
2057
2058
2059 class EmptyQuerySet(metaclass=InstanceCheckMeta):
2060 """
2061 Marker class to checking if a queryset is empty by .none():
2062 isinstance(qs.none(), EmptyQuerySet) -> True
2063 """
2064
2065 def __init__(self, *args, **kwargs):
2066 raise TypeError("EmptyQuerySet can't be instantiated")
2067
2068
2069 class RawQuerySet:
2070 """
2071 Provide an iterator which converts the results of raw SQL queries into
2072 annotated model instances.
2073 """
2074
2075 def __init__(
2076 self,
2077 raw_query,
2078 model=None,
2079 query=None,
2080 params=(),
2081 translations=None,
2082 using=None,
2083 hints=None,
2084 ):
2085 self.raw_query = raw_query
2086 self.model = model
2087 self._db = using
2088 self._hints = hints or {}
2089 self.query = query or sql.RawQuery(sql=raw_query, using=self.db, params=params)
2090 self.params = params
2091 self.translations = translations or {}
2092 self._result_cache = None
2093 self._prefetch_related_lookups = ()
2094 self._prefetch_done = False
2095
2096 def resolve_model_init_order(self):
2097 """Resolve the init field names and value positions."""
2098 converter = connections[self.db].introspection.identifier_converter
2099 model_init_fields = [
2100 f for f in self.model._meta.fields if converter(f.column) in self.columns
2101 ]
2102 annotation_fields = [
2103 (column, pos)
2104 for pos, column in enumerate(self.columns)
2105 if column not in self.model_fields
2106 ]
2107 model_init_order = [
2108 self.columns.index(converter(f.column)) for f in model_init_fields
2109 ]
2110 model_init_names = [f.attname for f in model_init_fields]
2111 return model_init_names, model_init_order, annotation_fields
2112
2113 def prefetch_related(self, *lookups):
2114 """Same as QuerySet.prefetch_related()"""
2115 clone = self._clone()
2116 if lookups == (None,):
2117 clone._prefetch_related_lookups = ()
2118 else:
2119 clone._prefetch_related_lookups = clone._prefetch_related_lookups + lookups
2120 return clone
2121
2122 def _prefetch_related_objects(self):
2123 prefetch_related_objects(self._result_cache, *self._prefetch_related_lookups)
2124 self._prefetch_done = True
2125
2126 def _clone(self):
2127 """Same as QuerySet._clone()"""
2128 c = self.__class__(
2129 self.raw_query,
2130 model=self.model,
2131 query=self.query,
2132 params=self.params,
2133 translations=self.translations,
2134 using=self._db,
2135 hints=self._hints,
2136 )
2137 c._prefetch_related_lookups = self._prefetch_related_lookups[:]
2138 return c
2139
2140 def _fetch_all(self):
2141 if self._result_cache is None:
2142 self._result_cache = list(self.iterator())
2143 if self._prefetch_related_lookups and not self._prefetch_done:
2144 self._prefetch_related_objects()
2145
2146 def __len__(self):
2147 self._fetch_all()
2148 return len(self._result_cache)
2149
2150 def __bool__(self):
2151 self._fetch_all()
2152 return bool(self._result_cache)
2153
2154 def __iter__(self):
2155 self._fetch_all()
2156 return iter(self._result_cache)
2157
2158 def __aiter__(self):
2159 # Remember, __aiter__ itself is synchronous, it's the thing it returns
2160 # that is async!
2161 async def generator():
2162 await sync_to_async(self._fetch_all)()
2163 for item in self._result_cache:
2164 yield item
2165
2166 return generator()
2167
2168 def iterator(self):
2169 yield from RawModelIterable(self)
2170
2171 def __repr__(self):
2172 return "<%s: %s>" % (self.__class__.__name__, self.query)
2173
2174 def __getitem__(self, k):
2175 return list(self)[k]
2176
2177 @property
2178 def db(self):
2179 """Return the database used if this query is executed now."""
2180 return self._db or router.db_for_read(self.model, **self._hints)
2181
2182 def using(self, alias):
2183 """Select the database this RawQuerySet should execute against."""
2184 return RawQuerySet(
2185 self.raw_query,
2186 model=self.model,
2187 query=self.query.chain(using=alias),
2188 params=self.params,
2189 translations=self.translations,
2190 using=alias,
2191 )
2192
2193 @cached_property
2194 def columns(self):
2195 """
2196 A list of model field names in the order they'll appear in the
2197 query results.
2198 """
2199 columns = self.query.get_columns()
2200 # Adjust any column names which don't match field names
2201 for query_name, model_name in self.translations.items():
2202 # Ignore translations for nonexistent column names
2203 try:
2204 index = columns.index(query_name)
2205 except ValueError:
2206 pass
2207 else:
2208 columns[index] = model_name
2209 return columns
2210
2211 @cached_property
2212 def model_fields(self):
2213 """A dict mapping column names to model field names."""
2214 converter = connections[self.db].introspection.identifier_converter
2215 model_fields = {}
2216 for field in self.model._meta.fields:
2217 model_fields[converter(field.column)] = field
2218 return model_fields
2219
2220
2221 class Prefetch:
2222 def __init__(self, lookup, queryset=None, to_attr=None):
2223 # `prefetch_through` is the path we traverse to perform the prefetch.
2224 self.prefetch_through = lookup
2225 # `prefetch_to` is the path to the attribute that stores the result.
2226 self.prefetch_to = lookup
2227 if queryset is not None and (
2228 isinstance(queryset, RawQuerySet)
2229 or (
2230 hasattr(queryset, "_iterable_class")
2231 and not issubclass(queryset._iterable_class, ModelIterable)
2232 )
2233 ):
2234 raise ValueError(
2235 "Prefetch querysets cannot use raw(), values(), and values_list()."
2236 )
2237 if to_attr:
2238 self.prefetch_to = LOOKUP_SEP.join(
2239 lookup.split(LOOKUP_SEP)[:-1] + [to_attr]
2240 )
2241
2242 self.queryset = queryset
2243 self.to_attr = to_attr
2244
2245 def __getstate__(self):
2246 obj_dict = self.__dict__.copy()
2247 if self.queryset is not None:
2248 queryset = self.queryset._chain()
2249 # Prevent the QuerySet from being evaluated
2250 queryset._result_cache = []
2251 queryset._prefetch_done = True
2252 obj_dict["queryset"] = queryset
2253 return obj_dict
2254
2255 def add_prefix(self, prefix):
2256 self.prefetch_through = prefix + LOOKUP_SEP + self.prefetch_through
2257 self.prefetch_to = prefix + LOOKUP_SEP + self.prefetch_to
2258
2259 def get_current_prefetch_to(self, level):
2260 return LOOKUP_SEP.join(self.prefetch_to.split(LOOKUP_SEP)[: level + 1])
2261
2262 def get_current_to_attr(self, level):
2263 parts = self.prefetch_to.split(LOOKUP_SEP)
2264 to_attr = parts[level]
2265 as_attr = self.to_attr and level == len(parts) - 1
2266 return to_attr, as_attr
2267
2268 def get_current_queryset(self, level):
2269 warnings.warn(
2270 "Prefetch.get_current_queryset() is deprecated. Use "
2271 "get_current_querysets() instead.",
2272 RemovedInDjango60Warning,
2273 stacklevel=2,
2274 )
2275 querysets = self.get_current_querysets(level)
2276 return querysets[0] if querysets is not None else None
2277
2278 def get_current_querysets(self, level):
2279 if (
2280 self.get_current_prefetch_to(level) == self.prefetch_to
2281 and self.queryset is not None
2282 ):
2283 return [self.queryset]
2284 return None
2285
2286 def __eq__(self, other):
2287 if not isinstance(other, Prefetch):
2288 return NotImplemented
2289 return self.prefetch_to == other.prefetch_to
2290
2291 def __hash__(self):
2292 return hash((self.__class__, self.prefetch_to))
2293
2294
2295 def normalize_prefetch_lookups(lookups, prefix=None):
2296 """Normalize lookups into Prefetch objects."""
2297 ret = []
2298 for lookup in lookups:
2299 if not isinstance(lookup, Prefetch):
2300 lookup = Prefetch(lookup)
2301 if prefix:
2302 lookup.add_prefix(prefix)
2303 ret.append(lookup)
2304 return ret
2305
2306
2307 def prefetch_related_objects(model_instances, *related_lookups):
2308 """
2309 Populate prefetched object caches for a list of model instances based on
2310 the lookups/Prefetch instances given.
2311 """
2312 if not model_instances:
2313 return # nothing to do
2314
2315 # We need to be able to dynamically add to the list of prefetch_related
2316 # lookups that we look up (see below). So we need some book keeping to
2317 # ensure we don't do duplicate work.
2318 done_queries = {} # dictionary of things like 'foo__bar': [results]
2319
2320 auto_lookups = set() # we add to this as we go through.
2321 followed_descriptors = set() # recursion protection
2322
2323 all_lookups = normalize_prefetch_lookups(reversed(related_lookups))
2324 while all_lookups:
2325 lookup = all_lookups.pop()
2326 if lookup.prefetch_to in done_queries:
2327 if lookup.queryset is not None:
2328 raise ValueError(
2329 "'%s' lookup was already seen with a different queryset. "
2330 "You may need to adjust the ordering of your lookups."
2331 % lookup.prefetch_to
2332 )
2333
2334 continue
2335
2336 # Top level, the list of objects to decorate is the result cache
2337 # from the primary QuerySet. It won't be for deeper levels.
2338 obj_list = model_instances
2339
2340 through_attrs = lookup.prefetch_through.split(LOOKUP_SEP)
2341 for level, through_attr in enumerate(through_attrs):
2342 # Prepare main instances
2343 if not obj_list:
2344 break
2345
2346 prefetch_to = lookup.get_current_prefetch_to(level)
2347 if prefetch_to in done_queries:
2348 # Skip any prefetching, and any object preparation
2349 obj_list = done_queries[prefetch_to]
2350 continue
2351
2352 # Prepare objects:
2353 good_objects = True
2354 for obj in obj_list:
2355 # Since prefetching can re-use instances, it is possible to have
2356 # the same instance multiple times in obj_list, so obj might
2357 # already be prepared.
2358 if not hasattr(obj, "_prefetched_objects_cache"):
2359 try:
2360 obj._prefetched_objects_cache = {}
2361 except (AttributeError, TypeError):
2362 # Must be an immutable object from
2363 # values_list(flat=True), for example (TypeError) or
2364 # a QuerySet subclass that isn't returning Model
2365 # instances (AttributeError), either in Django or a 3rd
2366 # party. prefetch_related() doesn't make sense, so quit.
2367 good_objects = False
2368 break
2369 if not good_objects:
2370 break
2371
2372 # Descend down tree
2373
2374 # We assume that objects retrieved are homogeneous (which is the premise
2375 # of prefetch_related), so what applies to first object applies to all.
2376 first_obj = obj_list[0]
2377 to_attr = lookup.get_current_to_attr(level)[0]
2378 prefetcher, descriptor, attr_found, is_fetched = get_prefetcher(
2379 first_obj, through_attr, to_attr
2380 )
2381
2382 if not attr_found:
2383 raise AttributeError(
2384 "Cannot find '%s' on %s object, '%s' is an invalid "
2385 "parameter to prefetch_related()"
2386 % (
2387 through_attr,
2388 first_obj.__class__.__name__,
2389 lookup.prefetch_through,
2390 )
2391 )
2392
2393 if level == len(through_attrs) - 1 and prefetcher is None:
2394 # Last one, this *must* resolve to something that supports
2395 # prefetching, otherwise there is no point adding it and the
2396 # developer asking for it has made a mistake.
2397 raise ValueError(
2398 "'%s' does not resolve to an item that supports "
2399 "prefetching - this is an invalid parameter to "
2400 "prefetch_related()." % lookup.prefetch_through
2401 )
2402
2403 obj_to_fetch = None
2404 if prefetcher is not None:
2405 obj_to_fetch = [obj for obj in obj_list if not is_fetched(obj)]
2406
2407 if obj_to_fetch:
2408 obj_list, additional_lookups = prefetch_one_level(
2409 obj_to_fetch,
2410 prefetcher,
2411 lookup,
2412 level,
2413 )
2414 # We need to ensure we don't keep adding lookups from the
2415 # same relationships to stop infinite recursion. So, if we
2416 # are already on an automatically added lookup, don't add
2417 # the new lookups from relationships we've seen already.
2418 if not (
2419 prefetch_to in done_queries
2420 and lookup in auto_lookups
2421 and descriptor in followed_descriptors
2422 ):
2423 done_queries[prefetch_to] = obj_list
2424 new_lookups = normalize_prefetch_lookups(
2425 reversed(additional_lookups), prefetch_to
2426 )
2427 auto_lookups.update(new_lookups)
2428 all_lookups.extend(new_lookups)
2429 followed_descriptors.add(descriptor)
2430 else:
2431 # Either a singly related object that has already been fetched
2432 # (e.g. via select_related), or hopefully some other property
2433 # that doesn't support prefetching but needs to be traversed.
2434
2435 # We replace the current list of parent objects with the list
2436 # of related objects, filtering out empty or missing values so
2437 # that we can continue with nullable or reverse relations.
2438 new_obj_list = []
2439 for obj in obj_list:
2440 if through_attr in getattr(obj, "_prefetched_objects_cache", ()):
2441 # If related objects have been prefetched, use the
2442 # cache rather than the object's through_attr.
2443 new_obj = list(obj._prefetched_objects_cache.get(through_attr))
2444 else:
2445 try:
2446 new_obj = getattr(obj, through_attr)
2447 except exceptions.ObjectDoesNotExist:
2448 continue
2449 if new_obj is None:
2450 continue
2451 # We special-case `list` rather than something more generic
2452 # like `Iterable` because we don't want to accidentally match
2453 # user models that define __iter__.
2454 if isinstance(new_obj, list):
2455 new_obj_list.extend(new_obj)
2456 else:
2457 new_obj_list.append(new_obj)
2458 obj_list = new_obj_list
2459
2460
2461 async def aprefetch_related_objects(model_instances, *related_lookups):
2462 """See prefetch_related_objects()."""
2463 return await sync_to_async(prefetch_related_objects)(
2464 model_instances, *related_lookups
2465 )
2466
2467
2468 def get_prefetcher(instance, through_attr, to_attr):
2469 """
2470 For the attribute 'through_attr' on the given instance, find
2471 an object that has a get_prefetch_querysets().
2472 Return a 4 tuple containing:
2473 (the object with get_prefetch_querysets (or None),
2474 the descriptor object representing this relationship (or None),
2475 a boolean that is False if the attribute was not found at all,
2476 a function that takes an instance and returns a boolean that is True if
2477 the attribute has already been fetched for that instance)
2478 """
2479
2480 def is_to_attr_fetched(model, to_attr):
2481 # Special case cached_property instances because hasattr() triggers
2482 # attribute computation and assignment.
2483 if isinstance(getattr(model, to_attr, None), cached_property):
2484
2485 def has_cached_property(instance):
2486 return to_attr in instance.__dict__
2487
2488 return has_cached_property
2489
2490 def has_to_attr_attribute(instance):
2491 return hasattr(instance, to_attr)
2492
2493 return has_to_attr_attribute
2494
2495 prefetcher = None
2496 is_fetched = is_to_attr_fetched(instance.__class__, to_attr)
2497
2498 # For singly related objects, we have to avoid getting the attribute
2499 # from the object, as this will trigger the query. So we first try
2500 # on the class, in order to get the descriptor object.
2501 rel_obj_descriptor = getattr(instance.__class__, through_attr, None)
2502 if rel_obj_descriptor is None:
2503 attr_found = hasattr(instance, through_attr)
2504 else:
2505 attr_found = True
2506 if rel_obj_descriptor:
2507 # singly related object, descriptor object has the
2508 # get_prefetch_querysets() method.
2509 if (
2510 hasattr(rel_obj_descriptor, "get_prefetch_querysets")
2511 # RemovedInDjango60Warning.
2512 or hasattr(rel_obj_descriptor, "get_prefetch_queryset")
2513 ):
2514 prefetcher = rel_obj_descriptor
2515 # If to_attr is set, check if the value has already been set,
2516 # which is done with has_to_attr_attribute(). Do not use the
2517 # method from the descriptor, as the cache_name it defines
2518 # checks the field name, not the to_attr value.
2519 if through_attr == to_attr:
2520 is_fetched = rel_obj_descriptor.is_cached
2521 else:
2522 # descriptor doesn't support prefetching, so we go ahead and get
2523 # the attribute on the instance rather than the class to
2524 # support many related managers
2525 rel_obj = getattr(instance, through_attr)
2526 if (
2527 hasattr(rel_obj, "get_prefetch_querysets")
2528 # RemovedInDjango60Warning.
2529 or hasattr(rel_obj, "get_prefetch_queryset")
2530 ):
2531 prefetcher = rel_obj
2532 if through_attr == to_attr:
2533
2534 def in_prefetched_cache(instance):
2535 return through_attr in instance._prefetched_objects_cache
2536
2537 is_fetched = in_prefetched_cache
2538 return prefetcher, rel_obj_descriptor, attr_found, is_fetched
2539
2540
2541 def prefetch_one_level(instances, prefetcher, lookup, level):
2542 """
2543 Helper function for prefetch_related_objects().
2544
2545 Run prefetches on all instances using the prefetcher object,
2546 assigning results to relevant caches in instance.
2547
2548 Return the prefetched objects along with any additional prefetches that
2549 must be done due to prefetch_related lookups found from default managers.
2550 """
2551 # prefetcher must have a method get_prefetch_querysets() which takes a list
2552 # of instances, and returns a tuple:
2553
2554 # (queryset of instances of self.model that are related to passed in instances,
2555 # callable that gets value to be matched for returned instances,
2556 # callable that gets value to be matched for passed in instances,
2557 # boolean that is True for singly related objects,
2558 # cache or field name to assign to,
2559 # boolean that is True when the previous argument is a cache name vs a field name).
2560
2561 # The 'values to be matched' must be hashable as they will be used
2562 # in a dictionary.
2563
2564 if hasattr(prefetcher, "get_prefetch_querysets"):
2565 (
2566 rel_qs,
2567 rel_obj_attr,
2568 instance_attr,
2569 single,
2570 cache_name,
2571 is_descriptor,
2572 ) = prefetcher.get_prefetch_querysets(
2573 instances, lookup.get_current_querysets(level)
2574 )
2575 else:
2576 warnings.warn(
2577 "The usage of get_prefetch_queryset() in prefetch_related_objects() is "
2578 "deprecated. Implement get_prefetch_querysets() instead.",
2579 RemovedInDjango60Warning,
2580 stacklevel=2,
2581 )
2582 queryset = None
2583 if querysets := lookup.get_current_querysets(level):
2584 queryset = querysets[0]
2585 (
2586 rel_qs,
2587 rel_obj_attr,
2588 instance_attr,
2589 single,
2590 cache_name,
2591 is_descriptor,
2592 ) = prefetcher.get_prefetch_queryset(instances, queryset)
2593 # We have to handle the possibility that the QuerySet we just got back
2594 # contains some prefetch_related lookups. We don't want to trigger the
2595 # prefetch_related functionality by evaluating the query. Rather, we need
2596 # to merge in the prefetch_related lookups.
2597 # Copy the lookups in case it is a Prefetch object which could be reused
2598 # later (happens in nested prefetch_related).
2599 additional_lookups = [
2600 copy.copy(additional_lookup)
2601 for additional_lookup in getattr(rel_qs, "_prefetch_related_lookups", ())
2602 ]
2603 if additional_lookups:
2604 # Don't need to clone because the manager should have given us a fresh
2605 # instance, so we access an internal instead of using public interface
2606 # for performance reasons.
2607 rel_qs._prefetch_related_lookups = ()
2608
2609 all_related_objects = list(rel_qs)
2610
2611 rel_obj_cache = {}
2612 for rel_obj in all_related_objects:
2613 rel_attr_val = rel_obj_attr(rel_obj)
2614 rel_obj_cache.setdefault(rel_attr_val, []).append(rel_obj)
2615
2616 to_attr, as_attr = lookup.get_current_to_attr(level)
2617 # Make sure `to_attr` does not conflict with a field.
2618 if as_attr and instances:
2619 # We assume that objects retrieved are homogeneous (which is the premise
2620 # of prefetch_related), so what applies to first object applies to all.
2621 model = instances[0].__class__
2622 try:
2623 model._meta.get_field(to_attr)
2624 except exceptions.FieldDoesNotExist:
2625 pass
2626 else:
2627 msg = "to_attr={} conflicts with a field on the {} model."
2628 raise ValueError(msg.format(to_attr, model.__name__))
2629
2630 # Whether or not we're prefetching the last part of the lookup.
2631 leaf = len(lookup.prefetch_through.split(LOOKUP_SEP)) - 1 == level
2632
2633 for obj in instances:
2634 instance_attr_val = instance_attr(obj)
2635 vals = rel_obj_cache.get(instance_attr_val, [])
2636
2637 if single:
2638 val = vals[0] if vals else None
2639 if as_attr:
2640 # A to_attr has been given for the prefetch.
2641 setattr(obj, to_attr, val)
2642 elif is_descriptor:
2643 # cache_name points to a field name in obj.
2644 # This field is a descriptor for a related object.
2645 setattr(obj, cache_name, val)
2646 else:
2647 # No to_attr has been given for this prefetch operation and the
2648 # cache_name does not point to a descriptor. Store the value of
2649 # the field in the object's field cache.
2650 obj._state.fields_cache[cache_name] = val
2651 else:
2652 if as_attr:
2653 setattr(obj, to_attr, vals)
2654 else:
2655 manager = getattr(obj, to_attr)
2656 if leaf and lookup.queryset is not None:
2657 qs = manager._apply_rel_filters(lookup.queryset)
2658 else:
2659 qs = manager.get_queryset()
2660 qs._result_cache = vals
2661 # We don't want the individual qs doing prefetch_related now,
2662 # since we have merged this into the current work.
2663 qs._prefetch_done = True
2664 obj._prefetched_objects_cache[cache_name] = qs
2665 return all_related_objects, additional_lookups
2666
2667
2668 class RelatedPopulator:
2669 """
2670 RelatedPopulator is used for select_related() object instantiation.
2671
2672 The idea is that each select_related() model will be populated by a
2673 different RelatedPopulator instance. The RelatedPopulator instances get
2674 klass_info and select (computed in SQLCompiler) plus the used db as
2675 input for initialization. That data is used to compute which columns
2676 to use, how to instantiate the model, and how to populate the links
2677 between the objects.
2678
2679 The actual creation of the objects is done in populate() method. This
2680 method gets row and from_obj as input and populates the select_related()
2681 model instance.
2682 """
2683
2684 def __init__(self, klass_info, select, db):
2685 self.db = db
2686 # Pre-compute needed attributes. The attributes are:
2687 # - model_cls: the possibly deferred model class to instantiate
2688 # - either:
2689 # - cols_start, cols_end: usually the columns in the row are
2690 # in the same order model_cls.__init__ expects them, so we
2691 # can instantiate by model_cls(*row[cols_start:cols_end])
2692 # - reorder_for_init: When select_related descends to a child
2693 # class, then we want to reuse the already selected parent
2694 # data. However, in this case the parent data isn't necessarily
2695 # in the same order that Model.__init__ expects it to be, so
2696 # we have to reorder the parent data. The reorder_for_init
2697 # attribute contains a function used to reorder the field data
2698 # in the order __init__ expects it.
2699 # - pk_idx: the index of the primary key field in the reordered
2700 # model data. Used to check if a related object exists at all.
2701 # - init_list: the field attnames fetched from the database. For
2702 # deferred models this isn't the same as all attnames of the
2703 # model's fields.
2704 # - related_populators: a list of RelatedPopulator instances if
2705 # select_related() descends to related models from this model.
2706 # - local_setter, remote_setter: Methods to set cached values on
2707 # the object being populated and on the remote object. Usually
2708 # these are Field.set_cached_value() methods.
2709 select_fields = klass_info["select_fields"]
2710 from_parent = klass_info["from_parent"]
2711 if not from_parent:
2712 self.cols_start = select_fields[0]
2713 self.cols_end = select_fields[-1] + 1
2714 self.init_list = [
2715 f[0].target.attname for f in select[self.cols_start : self.cols_end]
2716 ]
2717 self.reorder_for_init = None
2718 else:
2719 attname_indexes = {
2720 select[idx][0].target.attname: idx for idx in select_fields
2721 }
2722 model_init_attnames = (
2723 f.attname for f in klass_info["model"]._meta.concrete_fields
2724 )
2725 self.init_list = [
2726 attname for attname in model_init_attnames if attname in attname_indexes
2727 ]
2728 self.reorder_for_init = operator.itemgetter(
2729 *[attname_indexes[attname] for attname in self.init_list]
2730 )
2731
2732 self.model_cls = klass_info["model"]
2733 # A primary key must have all of its constituents not-NULL as
2734 # NULL != NULL and thus NULL cannot be referenced through a foreign
2735 # relationship. Therefore checking for a single member of the primary
2736 # key is enough to determine if the referenced object exists or not.
2737 self.pk_idx = self.init_list.index(self.model_cls._meta.pk_fields[0].attname)
2738 self.related_populators = get_related_populators(klass_info, select, self.db)
2739 self.local_setter = klass_info["local_setter"]
2740 self.remote_setter = klass_info["remote_setter"]
2741
2742 def populate(self, row, from_obj):
2743 if self.reorder_for_init:
2744 obj_data = self.reorder_for_init(row)
2745 else:
2746 obj_data = row[self.cols_start : self.cols_end]
2747 if obj_data[self.pk_idx] is None:
2748 obj = None
2749 else:
2750 obj = self.model_cls.from_db(self.db, self.init_list, obj_data)
2751 for rel_iter in self.related_populators:
2752 rel_iter.populate(row, obj)
2753 self.local_setter(from_obj, obj)
2754 if obj is not None:
2755 self.remote_setter(obj, from_obj)
2756
2757
2758 def get_related_populators(klass_info, select, db):
2759 iterators = []
2760 related_klass_infos = klass_info.get("related_klass_infos", [])
2761 for rel_klass_info in related_klass_infos:
2762 rel_cls = RelatedPopulator(rel_klass_info, select, db)
2763 iterators.append(rel_cls)
2764 return iterators