Debian Codegraph @

django/db/models/fields/json.py

raw
1 import json
2
3 from django import forms
4 from django.core import checks, exceptions
5 from django.db import NotSupportedError, connections, router
6 from django.db.models import expressions, lookups
7 from django.db.models.constants import LOOKUP_SEP
8 from django.db.models.fields import TextField
9 from django.db.models.lookups import (
10 FieldGetDbPrepValueMixin,
11 PostgresOperatorLookup,
12 Transform,
13 )
14 from django.utils.translation import gettext_lazy as _
15
16 from . import Field
17 from .mixins import CheckFieldDefaultMixin
18
19 __all__ = ["JSONField"]
20
21
22 class JSONField(CheckFieldDefaultMixin, Field):
23 empty_strings_allowed = False
24 description = _("A JSON object")
25 default_error_messages = {
26 "invalid": _("Value must be valid JSON."),
27 }
28 _default_hint = ("dict", "{}")
29
30 def __init__(
31 self,
32 verbose_name=None,
33 name=None,
34 encoder=None,
35 decoder=None,
36 **kwargs,
37 ):
38 if encoder and not callable(encoder):
39 raise ValueError("The encoder parameter must be a callable object.")
40 if decoder and not callable(decoder):
41 raise ValueError("The decoder parameter must be a callable object.")
42 self.encoder = encoder
43 self.decoder = decoder
44 super().__init__(verbose_name, name, **kwargs)
45
46 def check(self, **kwargs):
47 errors = super().check(**kwargs)
48 databases = kwargs.get("databases") or []
49 errors.extend(self._check_supported(databases))
50 return errors
51
52 def _check_supported(self, databases):
53 errors = []
54 for db in databases:
55 if not router.allow_migrate_model(db, self.model):
56 continue
57 connection = connections[db]
58 if (
59 self.model._meta.required_db_vendor
60 and self.model._meta.required_db_vendor != connection.vendor
61 ):
62 continue
63 if not (
64 "supports_json_field" in self.model._meta.required_db_features
65 or connection.features.supports_json_field
66 ):
67 errors.append(
68 checks.Error(
69 "%s does not support JSONFields." % connection.display_name,
70 obj=self.model,
71 id="fields.E180",
72 )
73 )
74 return errors
75
76 def deconstruct(self):
77 name, path, args, kwargs = super().deconstruct()
78 if self.encoder is not None:
79 kwargs["encoder"] = self.encoder
80 if self.decoder is not None:
81 kwargs["decoder"] = self.decoder
82 return name, path, args, kwargs
83
84 def from_db_value(self, value, expression, connection):
85 if value is None:
86 return value
87 # Some backends (SQLite at least) extract non-string values in their
88 # SQL datatypes.
89 if isinstance(expression, KeyTransform) and not isinstance(value, str):
90 return value
91 try:
92 return json.loads(value, cls=self.decoder)
93 except json.JSONDecodeError:
94 return value
95
96 def get_internal_type(self):
97 return "JSONField"
98
99 def get_db_prep_value(self, value, connection, prepared=False):
100 if not prepared:
101 value = self.get_prep_value(value)
102 return connection.ops.adapt_json_value(value, self.encoder)
103
104 def get_db_prep_save(self, value, connection):
105 # This slightly involved logic is to allow for `None` to be used to
106 # store SQL `NULL` while `Value(None, JSONField())` can be used to
107 # store JSON `null` while preventing compilable `as_sql` values from
108 # making their way to `get_db_prep_value`, which is what the `super()`
109 # implementation does.
110 if value is None:
111 return value
112 if (
113 isinstance(value, expressions.Value)
114 and value.value is None
115 and isinstance(value.output_field, JSONField)
116 ):
117 value = None
118 return super().get_db_prep_save(value, connection)
119
120 def get_transform(self, name):
121 transform = super().get_transform(name)
122 if transform:
123 return transform
124 return KeyTransformFactory(name)
125
126 def validate(self, value, model_instance):
127 super().validate(value, model_instance)
128 try:
129 json.dumps(value, cls=self.encoder)
130 except TypeError:
131 raise exceptions.ValidationError(
132 self.error_messages["invalid"],
133 code="invalid",
134 params={"value": value},
135 )
136
137 def value_to_string(self, obj):
138 return self.value_from_object(obj)
139
140 def formfield(self, **kwargs):
141 return super().formfield(
142 **{
143 "form_class": forms.JSONField,
144 "encoder": self.encoder,
145 "decoder": self.decoder,
146 **kwargs,
147 }
148 )
149
150
151 def compile_json_path(key_transforms, include_root=True):
152 path = ["$"] if include_root else []
153 for key_transform in key_transforms:
154 try:
155 num = int(key_transform)
156 except ValueError: # non-integer
157 path.append(".")
158 path.append(json.dumps(key_transform))
159 else:
160 path.append("[%s]" % num)
161 return "".join(path)
162
163
164 class DataContains(FieldGetDbPrepValueMixin, PostgresOperatorLookup):
165 lookup_name = "contains"
166 postgres_operator = "@>"
167
168 def as_sql(self, compiler, connection):
169 if not connection.features.supports_json_field_contains:
170 raise NotSupportedError(
171 "contains lookup is not supported on this database backend."
172 )
173 lhs, lhs_params = self.process_lhs(compiler, connection)
174 rhs, rhs_params = self.process_rhs(compiler, connection)
175 params = tuple(lhs_params) + tuple(rhs_params)
176 return "JSON_CONTAINS(%s, %s)" % (lhs, rhs), params
177
178
179 class ContainedBy(FieldGetDbPrepValueMixin, PostgresOperatorLookup):
180 lookup_name = "contained_by"
181 postgres_operator = "<@"
182
183 def as_sql(self, compiler, connection):
184 if not connection.features.supports_json_field_contains:
185 raise NotSupportedError(
186 "contained_by lookup is not supported on this database backend."
187 )
188 lhs, lhs_params = self.process_lhs(compiler, connection)
189 rhs, rhs_params = self.process_rhs(compiler, connection)
190 params = tuple(rhs_params) + tuple(lhs_params)
191 return "JSON_CONTAINS(%s, %s)" % (rhs, lhs), params
192
193
194 class HasKeyLookup(PostgresOperatorLookup):
195 logical_operator = None
196
197 def compile_json_path_final_key(self, key_transform):
198 # Compile the final key without interpreting ints as array elements.
199 return ".%s" % json.dumps(key_transform)
200
201 def _as_sql_parts(self, compiler, connection):
202 # Process JSON path from the left-hand side.
203 if isinstance(self.lhs, KeyTransform):
204 lhs_sql, lhs_params, lhs_key_transforms = self.lhs.preprocess_lhs(
205 compiler, connection
206 )
207 lhs_json_path = compile_json_path(lhs_key_transforms)
208 else:
209 lhs_sql, lhs_params = self.process_lhs(compiler, connection)
210 lhs_json_path = "$"
211 # Process JSON path from the right-hand side.
212 rhs = self.rhs
213 if not isinstance(rhs, (list, tuple)):
214 rhs = [rhs]
215 for key in rhs:
216 if isinstance(key, KeyTransform):
217 *_, rhs_key_transforms = key.preprocess_lhs(compiler, connection)
218 else:
219 rhs_key_transforms = [key]
220 *rhs_key_transforms, final_key = rhs_key_transforms
221 rhs_json_path = compile_json_path(rhs_key_transforms, include_root=False)
222 rhs_json_path += self.compile_json_path_final_key(final_key)
223 yield lhs_sql, lhs_params, lhs_json_path + rhs_json_path
224
225 def _combine_sql_parts(self, parts):
226 # Add condition for each key.
227 if self.logical_operator:
228 return "(%s)" % self.logical_operator.join(parts)
229 return "".join(parts)
230
231 def as_sql(self, compiler, connection, template=None):
232 sql_parts = []
233 params = []
234 for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(
235 compiler, connection
236 ):
237 sql_parts.append(template % (lhs_sql, "%s"))
238 params.extend(lhs_params + [rhs_json_path])
239 return self._combine_sql_parts(sql_parts), tuple(params)
240
241 def as_mysql(self, compiler, connection):
242 return self.as_sql(
243 compiler, connection, template="JSON_CONTAINS_PATH(%s, 'one', %s)"
244 )
245
246 def as_oracle(self, compiler, connection):
247 # Use a custom delimiter to prevent the JSON path from escaping the SQL
248 # literal. See comment in KeyTransform.
249 template = "JSON_EXISTS(%s, q'\uffff%s\uffff')"
250 sql_parts = []
251 params = []
252 for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(
253 compiler, connection
254 ):
255 # Add right-hand-side directly into SQL because it cannot be passed
256 # as bind variables to JSON_EXISTS. It might result in invalid
257 # queries but it is assumed that it cannot be evaded because the
258 # path is JSON serialized.
259 sql_parts.append(template % (lhs_sql, rhs_json_path))
260 params.extend(lhs_params)
261 return self._combine_sql_parts(sql_parts), tuple(params)
262
263 def as_postgresql(self, compiler, connection):
264 if isinstance(self.rhs, KeyTransform):
265 *_, rhs_key_transforms = self.rhs.preprocess_lhs(compiler, connection)
266 for key in rhs_key_transforms[:-1]:
267 self.lhs = KeyTransform(key, self.lhs)
268 self.rhs = rhs_key_transforms[-1]
269 return super().as_postgresql(compiler, connection)
270
271 def as_sqlite(self, compiler, connection):
272 return self.as_sql(
273 compiler, connection, template="JSON_TYPE(%s, %s) IS NOT NULL"
274 )
275
276
277 class HasKey(HasKeyLookup):
278 lookup_name = "has_key"
279 postgres_operator = "?"
280 prepare_rhs = False
281
282
283 class HasKeys(HasKeyLookup):
284 lookup_name = "has_keys"
285 postgres_operator = "?&"
286 logical_operator = " AND "
287
288 def get_prep_lookup(self):
289 return [str(item) for item in self.rhs]
290
291
292 class HasAnyKeys(HasKeys):
293 lookup_name = "has_any_keys"
294 postgres_operator = "?|"
295 logical_operator = " OR "
296
297
298 class HasKeyOrArrayIndex(HasKey):
299 def compile_json_path_final_key(self, key_transform):
300 return compile_json_path([key_transform], include_root=False)
301
302
303 class CaseInsensitiveMixin:
304 """
305 Mixin to allow case-insensitive comparison of JSON values on MySQL.
306 MySQL handles strings used in JSON context using the utf8mb4_bin collation.
307 Because utf8mb4_bin is a binary collation, comparison of JSON values is
308 case-sensitive.
309 """
310
311 def process_lhs(self, compiler, connection):
312 lhs, lhs_params = super().process_lhs(compiler, connection)
313 if connection.vendor == "mysql":
314 return "LOWER(%s)" % lhs, lhs_params
315 return lhs, lhs_params
316
317 def process_rhs(self, compiler, connection):
318 rhs, rhs_params = super().process_rhs(compiler, connection)
319 if connection.vendor == "mysql":
320 return "LOWER(%s)" % rhs, rhs_params
321 return rhs, rhs_params
322
323
324 class JSONExact(lookups.Exact):
325 can_use_none_as_rhs = True
326
327 def process_rhs(self, compiler, connection):
328 rhs, rhs_params = super().process_rhs(compiler, connection)
329 # Treat None lookup values as null.
330 if rhs == "%s" and rhs_params == [None]:
331 rhs_params = ["null"]
332 if connection.vendor == "mysql":
333 func = ["JSON_EXTRACT(%s, '$')"] * len(rhs_params)
334 rhs %= tuple(func)
335 return rhs, rhs_params
336
337 def as_oracle(self, compiler, connection):
338 lhs, lhs_params = self.process_lhs(compiler, connection)
339 rhs, rhs_params = self.process_rhs(compiler, connection)
340 if connection.features.supports_primitives_in_json_field:
341 lhs = f"JSON({lhs})"
342 rhs = f"JSON({rhs})"
343 return f"JSON_EQUAL({lhs}, {rhs} ERROR ON ERROR)", (*lhs_params, *rhs_params)
344
345
346 class JSONIContains(CaseInsensitiveMixin, lookups.IContains):
347 pass
348
349
350 JSONField.register_lookup(DataContains)
351 JSONField.register_lookup(ContainedBy)
352 JSONField.register_lookup(HasKey)
353 JSONField.register_lookup(HasKeys)
354 JSONField.register_lookup(HasAnyKeys)
355 JSONField.register_lookup(JSONExact)
356 JSONField.register_lookup(JSONIContains)
357
358
359 class KeyTransform(Transform):
360 postgres_operator = "->"
361 postgres_nested_operator = "#>"
362
363 def __init__(self, key_name, *args, **kwargs):
364 super().__init__(*args, **kwargs)
365 self.key_name = str(key_name)
366
367 def preprocess_lhs(self, compiler, connection):
368 key_transforms = [self.key_name]
369 previous = self.lhs
370 while isinstance(previous, KeyTransform):
371 key_transforms.insert(0, previous.key_name)
372 previous = previous.lhs
373 lhs, params = compiler.compile(previous)
374 if connection.vendor == "oracle":
375 # Escape string-formatting.
376 key_transforms = [key.replace("%", "%%") for key in key_transforms]
377 return lhs, params, key_transforms
378
379 def as_mysql(self, compiler, connection):
380 lhs, params, key_transforms = self.preprocess_lhs(compiler, connection)
381 json_path = compile_json_path(key_transforms)
382 return "JSON_EXTRACT(%s, %%s)" % lhs, tuple(params) + (json_path,)
383
384 def as_oracle(self, compiler, connection):
385 lhs, params, key_transforms = self.preprocess_lhs(compiler, connection)
386 json_path = compile_json_path(key_transforms)
387 if connection.features.supports_primitives_in_json_field:
388 sql = (
389 "COALESCE("
390 "JSON_VALUE(%s, q'\uffff%s\uffff'),"
391 "JSON_QUERY(%s, q'\uffff%s\uffff' DISALLOW SCALARS)"
392 ")"
393 )
394 else:
395 sql = (
396 "COALESCE("
397 "JSON_QUERY(%s, q'\uffff%s\uffff'),"
398 "JSON_VALUE(%s, q'\uffff%s\uffff')"
399 ")"
400 )
401 # Add paths directly into SQL because path expressions cannot be passed
402 # as bind variables on Oracle. Use a custom delimiter to prevent the
403 # JSON path from escaping the SQL literal. Each key in the JSON path is
404 # passed through json.dumps() with ensure_ascii=True (the default),
405 # which converts the delimiter into the escaped \uffff format. This
406 # ensures that the delimiter is not present in the JSON path.
407 return sql % ((lhs, json_path) * 2), tuple(params) * 2
408
409 def as_postgresql(self, compiler, connection):
410 lhs, params, key_transforms = self.preprocess_lhs(compiler, connection)
411 if len(key_transforms) > 1:
412 sql = "(%s %s %%s)" % (lhs, self.postgres_nested_operator)
413 return sql, tuple(params) + (key_transforms,)
414 try:
415 lookup = int(self.key_name)
416 except ValueError:
417 lookup = self.key_name
418 return "(%s %s %%s)" % (lhs, self.postgres_operator), tuple(params) + (lookup,)
419
420 def as_sqlite(self, compiler, connection):
421 lhs, params, key_transforms = self.preprocess_lhs(compiler, connection)
422 json_path = compile_json_path(key_transforms)
423 datatype_values = ",".join(
424 [repr(datatype) for datatype in connection.ops.jsonfield_datatype_values]
425 )
426 return (
427 "(CASE WHEN JSON_TYPE(%s, %%s) IN (%s) "
428 "THEN JSON_TYPE(%s, %%s) ELSE JSON_EXTRACT(%s, %%s) END)"
429 ) % (lhs, datatype_values, lhs, lhs), (tuple(params) + (json_path,)) * 3
430
431
432 class KeyTextTransform(KeyTransform):
433 postgres_operator = "->>"
434 postgres_nested_operator = "#>>"
435 output_field = TextField()
436
437 def as_mysql(self, compiler, connection):
438 if connection.mysql_is_mariadb:
439 # MariaDB doesn't support -> and ->> operators (see MDEV-13594).
440 sql, params = super().as_mysql(compiler, connection)
441 return "JSON_UNQUOTE(%s)" % sql, params
442 else:
443 lhs, params, key_transforms = self.preprocess_lhs(compiler, connection)
444 json_path = compile_json_path(key_transforms)
445 return "(%s ->> %%s)" % lhs, tuple(params) + (json_path,)
446
447 @classmethod
448 def from_lookup(cls, lookup):
449 transform, *keys = lookup.split(LOOKUP_SEP)
450 if not keys:
451 raise ValueError("Lookup must contain key or index transforms.")
452 for key in keys:
453 transform = cls(key, transform)
454 return transform
455
456
457 KT = KeyTextTransform.from_lookup
458
459
460 class KeyTransformTextLookupMixin:
461 """
462 Mixin for combining with a lookup expecting a text lhs from a JSONField
463 key lookup. On PostgreSQL, make use of the ->> operator instead of casting
464 key values to text and performing the lookup on the resulting
465 representation.
466 """
467
468 def __init__(self, key_transform, *args, **kwargs):
469 if not isinstance(key_transform, KeyTransform):
470 raise TypeError(
471 "Transform should be an instance of KeyTransform in order to "
472 "use this lookup."
473 )
474 key_text_transform = KeyTextTransform(
475 key_transform.key_name,
476 *key_transform.source_expressions,
477 **key_transform.extra,
478 )
479 super().__init__(key_text_transform, *args, **kwargs)
480
481
482 class KeyTransformIsNull(lookups.IsNull):
483 # key__isnull=False is the same as has_key='key'
484 def as_oracle(self, compiler, connection):
485 sql, params = HasKeyOrArrayIndex(
486 self.lhs.lhs,
487 self.lhs.key_name,
488 ).as_oracle(compiler, connection)
489 if not self.rhs:
490 return sql, params
491 # Column doesn't have a key or IS NULL.
492 lhs, lhs_params, _ = self.lhs.preprocess_lhs(compiler, connection)
493 return "(NOT %s OR %s IS NULL)" % (sql, lhs), tuple(params) + tuple(lhs_params)
494
495 def as_sqlite(self, compiler, connection):
496 template = "JSON_TYPE(%s, %s) IS NULL"
497 if not self.rhs:
498 template = "JSON_TYPE(%s, %s) IS NOT NULL"
499 return HasKeyOrArrayIndex(self.lhs.lhs, self.lhs.key_name).as_sql(
500 compiler,
501 connection,
502 template=template,
503 )
504
505
506 class KeyTransformIn(lookups.In):
507 def resolve_expression_parameter(self, compiler, connection, sql, param):
508 sql, params = super().resolve_expression_parameter(
509 compiler,
510 connection,
511 sql,
512 param,
513 )
514 if (
515 not hasattr(param, "as_sql")
516 and not connection.features.has_native_json_field
517 ):
518 if connection.vendor == "oracle":
519 value = json.loads(param)
520 sql = "%s(JSON_OBJECT('value' VALUE %%s FORMAT JSON), '$.value')"
521 if isinstance(value, (list, dict)):
522 sql %= "JSON_QUERY"
523 else:
524 sql %= "JSON_VALUE"
525 elif connection.vendor == "mysql" or (
526 connection.vendor == "sqlite"
527 and params[0] not in connection.ops.jsonfield_datatype_values
528 ):
529 sql = "JSON_EXTRACT(%s, '$')"
530 if connection.vendor == "mysql" and connection.mysql_is_mariadb:
531 sql = "JSON_UNQUOTE(%s)" % sql
532 return sql, params
533
534
535 class KeyTransformExact(JSONExact):
536 def process_rhs(self, compiler, connection):
537 if isinstance(self.rhs, KeyTransform):
538 return super(lookups.Exact, self).process_rhs(compiler, connection)
539 rhs, rhs_params = super().process_rhs(compiler, connection)
540 if connection.vendor == "oracle":
541 func = []
542 sql = "%s(JSON_OBJECT('value' VALUE %%s FORMAT JSON), '$.value')"
543 for value in rhs_params:
544 value = json.loads(value)
545 if isinstance(value, (list, dict)):
546 func.append(sql % "JSON_QUERY")
547 else:
548 func.append(sql % "JSON_VALUE")
549 rhs %= tuple(func)
550 elif connection.vendor == "sqlite":
551 func = []
552 for value in rhs_params:
553 if value in connection.ops.jsonfield_datatype_values:
554 func.append("%s")
555 else:
556 func.append("JSON_EXTRACT(%s, '$')")
557 rhs %= tuple(func)
558 return rhs, rhs_params
559
560 def as_oracle(self, compiler, connection):
561 rhs, rhs_params = super().process_rhs(compiler, connection)
562 if rhs_params == ["null"]:
563 # Field has key and it's NULL.
564 has_key_expr = HasKeyOrArrayIndex(self.lhs.lhs, self.lhs.key_name)
565 has_key_sql, has_key_params = has_key_expr.as_oracle(compiler, connection)
566 is_null_expr = self.lhs.get_lookup("isnull")(self.lhs, True)
567 is_null_sql, is_null_params = is_null_expr.as_sql(compiler, connection)
568 return (
569 "%s AND %s" % (has_key_sql, is_null_sql),
570 tuple(has_key_params) + tuple(is_null_params),
571 )
572 return super().as_sql(compiler, connection)
573
574
575 class KeyTransformIExact(
576 CaseInsensitiveMixin, KeyTransformTextLookupMixin, lookups.IExact
577 ):
578 pass
579
580
581 class KeyTransformIContains(
582 CaseInsensitiveMixin, KeyTransformTextLookupMixin, lookups.IContains
583 ):
584 pass
585
586
587 class KeyTransformStartsWith(KeyTransformTextLookupMixin, lookups.StartsWith):
588 pass
589
590
591 class KeyTransformIStartsWith(
592 CaseInsensitiveMixin, KeyTransformTextLookupMixin, lookups.IStartsWith
593 ):
594 pass
595
596
597 class KeyTransformEndsWith(KeyTransformTextLookupMixin, lookups.EndsWith):
598 pass
599
600
601 class KeyTransformIEndsWith(
602 CaseInsensitiveMixin, KeyTransformTextLookupMixin, lookups.IEndsWith
603 ):
604 pass
605
606
607 class KeyTransformRegex(KeyTransformTextLookupMixin, lookups.Regex):
608 pass
609
610
611 class KeyTransformIRegex(
612 CaseInsensitiveMixin, KeyTransformTextLookupMixin, lookups.IRegex
613 ):
614 pass
615
616
617 class KeyTransformNumericLookupMixin:
618 def process_rhs(self, compiler, connection):
619 rhs, rhs_params = super().process_rhs(compiler, connection)
620 if not connection.features.has_native_json_field:
621 rhs_params = [json.loads(value) for value in rhs_params]
622 return rhs, rhs_params
623
624
625 class KeyTransformLt(KeyTransformNumericLookupMixin, lookups.LessThan):
626 pass
627
628
629 class KeyTransformLte(KeyTransformNumericLookupMixin, lookups.LessThanOrEqual):
630 pass
631
632
633 class KeyTransformGt(KeyTransformNumericLookupMixin, lookups.GreaterThan):
634 pass
635
636
637 class KeyTransformGte(KeyTransformNumericLookupMixin, lookups.GreaterThanOrEqual):
638 pass
639
640
641 KeyTransform.register_lookup(KeyTransformIn)
642 KeyTransform.register_lookup(KeyTransformExact)
643 KeyTransform.register_lookup(KeyTransformIExact)
644 KeyTransform.register_lookup(KeyTransformIsNull)
645 KeyTransform.register_lookup(KeyTransformIContains)
646 KeyTransform.register_lookup(KeyTransformStartsWith)
647 KeyTransform.register_lookup(KeyTransformIStartsWith)
648 KeyTransform.register_lookup(KeyTransformEndsWith)
649 KeyTransform.register_lookup(KeyTransformIEndsWith)
650 KeyTransform.register_lookup(KeyTransformRegex)
651 KeyTransform.register_lookup(KeyTransformIRegex)
652
653 KeyTransform.register_lookup(KeyTransformLt)
654 KeyTransform.register_lookup(KeyTransformLte)
655 KeyTransform.register_lookup(KeyTransformGt)
656 KeyTransform.register_lookup(KeyTransformGte)
657
658
659 class KeyTransformFactory:
660 def __init__(self, key_name):
661 self.key_name = key_name
662
663 def __call__(self, *args, **kwargs):
664 return KeyTransform(self.key_name, *args, **kwargs)