Debian Codegraph @

django/core/serializers/xml_serializer.py

raw
1 """
2 XML serializer.
3 """
4
5 import json
6 from contextlib import contextmanager
7 from xml.dom import minidom, pulldom
8 from xml.sax import handler
9 from xml.sax.expatreader import ExpatParser as _ExpatParser
10
11 from django.apps import apps
12 from django.conf import settings
13 from django.core.exceptions import ObjectDoesNotExist
14 from django.core.serializers import base
15 from django.db import DEFAULT_DB_ALIAS, models
16 from django.utils.xmlutils import SimplerXMLGenerator, UnserializableContentError
17
18
19 @contextmanager
20 def fast_cache_clearing():
21 """Workaround for performance issues in minidom document checks.
22
23 Speeds up repeated DOM operations by skipping unnecessary full traversal
24 of the DOM tree.
25 """
26 module_helper_was_lambda = False
27 if original_fn := getattr(minidom, "_in_document", None):
28 module_helper_was_lambda = original_fn.__name__ == "<lambda>"
29 if not module_helper_was_lambda:
30 minidom._in_document = lambda node: bool(node.ownerDocument)
31 try:
32 yield
33 finally:
34 if original_fn and not module_helper_was_lambda:
35 minidom._in_document = original_fn
36
37
38 class Serializer(base.Serializer):
39 """Serialize a QuerySet to XML."""
40
41 def indent(self, level):
42 if self.options.get("indent") is not None:
43 self.xml.ignorableWhitespace(
44 "\n" + " " * self.options.get("indent") * level
45 )
46
47 def start_serialization(self):
48 """
49 Start serialization -- open the XML document and the root element.
50 """
51 self.xml = SimplerXMLGenerator(
52 self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET)
53 )
54 self.xml.startDocument()
55 self.xml.startElement("django-objects", {"version": "1.0"})
56
57 def end_serialization(self):
58 """
59 End serialization -- end the document.
60 """
61 self.indent(0)
62 self.xml.endElement("django-objects")
63 self.xml.endDocument()
64
65 def start_object(self, obj):
66 """
67 Called as each object is handled.
68 """
69 if not hasattr(obj, "_meta"):
70 raise base.SerializationError(
71 "Non-model object (%s) encountered during serialization" % type(obj)
72 )
73
74 self.indent(1)
75 attrs = {"model": str(obj._meta)}
76 if not self.use_natural_primary_keys or not hasattr(obj, "natural_key"):
77 obj_pk = obj.pk
78 if obj_pk is not None:
79 attrs["pk"] = obj._meta.pk.value_to_string(obj)
80
81 self.xml.startElement("object", attrs)
82
83 def end_object(self, obj):
84 """
85 Called after handling all fields for an object.
86 """
87 self.indent(1)
88 self.xml.endElement("object")
89
90 def handle_field(self, obj, field):
91 """
92 Handle each field on an object (except for ForeignKeys and
93 ManyToManyFields).
94 """
95 self.indent(2)
96 self.xml.startElement(
97 "field",
98 {
99 "name": field.name,
100 "type": field.get_internal_type(),
101 },
102 )
103
104 # Get a "string version" of the object's data.
105 if getattr(obj, field.name) is not None:
106 value = field.value_to_string(obj)
107 if field.get_internal_type() == "JSONField":
108 # Dump value since JSONField.value_to_string() doesn't output
109 # strings.
110 value = json.dumps(value, cls=field.encoder)
111 try:
112 self.xml.characters(value)
113 except UnserializableContentError:
114 raise ValueError(
115 "%s.%s (pk:%s) contains unserializable characters"
116 % (obj.__class__.__name__, field.name, obj.pk)
117 )
118 else:
119 self.xml.addQuickElement("None")
120
121 self.xml.endElement("field")
122
123 def handle_fk_field(self, obj, field):
124 """
125 Handle a ForeignKey (they need to be treated slightly
126 differently from regular fields).
127 """
128 self._start_relational_field(field)
129 related_att = getattr(obj, field.attname)
130 if related_att is not None:
131 if self.use_natural_foreign_keys and hasattr(
132 field.remote_field.model, "natural_key"
133 ):
134 related = getattr(obj, field.name)
135 # If related object has a natural key, use it
136 related = related.natural_key()
137 # Iterable natural keys are rolled out as subelements
138 for key_value in related:
139 self.xml.startElement("natural", {})
140 self.xml.characters(str(key_value))
141 self.xml.endElement("natural")
142 else:
143 self.xml.characters(str(related_att))
144 else:
145 self.xml.addQuickElement("None")
146 self.xml.endElement("field")
147
148 def handle_m2m_field(self, obj, field):
149 """
150 Handle a ManyToManyField. Related objects are only serialized as
151 references to the object's PK (i.e. the related *data* is not dumped,
152 just the relation).
153 """
154 if field.remote_field.through._meta.auto_created:
155 self._start_relational_field(field)
156 if self.use_natural_foreign_keys and hasattr(
157 field.remote_field.model, "natural_key"
158 ):
159 # If the objects in the m2m have a natural key, use it
160 def handle_m2m(value):
161 natural = value.natural_key()
162 # Iterable natural keys are rolled out as subelements
163 self.xml.startElement("object", {})
164 for key_value in natural:
165 self.xml.startElement("natural", {})
166 self.xml.characters(str(key_value))
167 self.xml.endElement("natural")
168 self.xml.endElement("object")
169
170 def queryset_iterator(obj, field):
171 attr = getattr(obj, field.name)
172 chunk_size = (
173 2000 if getattr(attr, "prefetch_cache_name", None) else None
174 )
175 return attr.iterator(chunk_size)
176
177 else:
178
179 def handle_m2m(value):
180 self.xml.addQuickElement("object", attrs={"pk": str(value.pk)})
181
182 def queryset_iterator(obj, field):
183 query_set = getattr(obj, field.name).select_related(None).only("pk")
184 chunk_size = 2000 if query_set._prefetch_related_lookups else None
185 return query_set.iterator(chunk_size=chunk_size)
186
187 m2m_iter = getattr(obj, "_prefetched_objects_cache", {}).get(
188 field.name,
189 queryset_iterator(obj, field),
190 )
191 for relobj in m2m_iter:
192 handle_m2m(relobj)
193
194 self.xml.endElement("field")
195
196 def _start_relational_field(self, field):
197 """Output the <field> element for relational fields."""
198 self.indent(2)
199 self.xml.startElement(
200 "field",
201 {
202 "name": field.name,
203 "rel": field.remote_field.__class__.__name__,
204 "to": str(field.remote_field.model._meta),
205 },
206 )
207
208
209 class Deserializer(base.Deserializer):
210 """Deserialize XML."""
211
212 def __init__(
213 self,
214 stream_or_string,
215 *,
216 using=DEFAULT_DB_ALIAS,
217 ignorenonexistent=False,
218 **options,
219 ):
220 super().__init__(stream_or_string, **options)
221 self.handle_forward_references = options.pop("handle_forward_references", False)
222 self.event_stream = pulldom.parse(self.stream, self._make_parser())
223 self.db = using
224 self.ignore = ignorenonexistent
225
226 def _make_parser(self):
227 """Create a hardened XML parser (no custom/external entities)."""
228 return DefusedExpatParser()
229
230 def __next__(self):
231 for event, node in self.event_stream:
232 if event == "START_ELEMENT" and node.nodeName == "object":
233 with fast_cache_clearing():
234 self.event_stream.expandNode(node)
235 return self._handle_object(node)
236 raise StopIteration
237
238 def _handle_object(self, node):
239 """Convert an <object> node to a DeserializedObject."""
240 # Look up the model using the model loading mechanism. If this fails,
241 # bail.
242 Model = self._get_model_from_node(node, "model")
243
244 # Start building a data dictionary from the object.
245 data = {}
246 if node.hasAttribute("pk"):
247 data[Model._meta.pk.attname] = Model._meta.pk.to_python(
248 node.getAttribute("pk")
249 )
250
251 # Also start building a dict of m2m data (this is saved as
252 # {m2m_accessor_attribute : [list_of_related_objects]})
253 m2m_data = {}
254 deferred_fields = {}
255
256 field_names = {f.name for f in Model._meta.get_fields()}
257 # Deserialize each field.
258 for field_node in node.getElementsByTagName("field"):
259 # If the field is missing the name attribute, bail (are you
260 # sensing a pattern here?)
261 field_name = field_node.getAttribute("name")
262 if not field_name:
263 raise base.DeserializationError(
264 "<field> node is missing the 'name' attribute"
265 )
266
267 # Get the field from the Model. This will raise a
268 # FieldDoesNotExist if, well, the field doesn't exist, which will
269 # be propagated correctly unless ignorenonexistent=True is used.
270 if self.ignore and field_name not in field_names:
271 continue
272 field = Model._meta.get_field(field_name)
273
274 # As is usually the case, relation fields get the special treatment.
275 if field.remote_field and isinstance(
276 field.remote_field, models.ManyToManyRel
277 ):
278 value = self._handle_m2m_field_node(field_node, field)
279 if value == base.DEFER_FIELD:
280 deferred_fields[field] = [
281 [
282 getInnerText(nat_node).strip()
283 for nat_node in obj_node.getElementsByTagName("natural")
284 ]
285 for obj_node in field_node.getElementsByTagName("object")
286 ]
287 else:
288 m2m_data[field.name] = value
289 elif field.remote_field and isinstance(
290 field.remote_field, models.ManyToOneRel
291 ):
292 value = self._handle_fk_field_node(field_node, field)
293 if value == base.DEFER_FIELD:
294 deferred_fields[field] = [
295 getInnerText(k).strip()
296 for k in field_node.getElementsByTagName("natural")
297 ]
298 else:
299 data[field.attname] = value
300 else:
301 if field_node.getElementsByTagName("None"):
302 value = None
303 else:
304 value = field.to_python(getInnerText(field_node).strip())
305 # Load value since JSONField.to_python() outputs strings.
306 if field.get_internal_type() == "JSONField":
307 value = json.loads(value, cls=field.decoder)
308 data[field.name] = value
309
310 obj = base.build_instance(Model, data, self.db)
311
312 # Return a DeserializedObject so that the m2m data has a place to live.
313 return base.DeserializedObject(obj, m2m_data, deferred_fields)
314
315 def _handle_fk_field_node(self, node, field):
316 """
317 Handle a <field> node for a ForeignKey
318 """
319 # Check if there is a child node named 'None', returning None if so.
320 if node.getElementsByTagName("None"):
321 return None
322 else:
323 model = field.remote_field.model
324 if hasattr(model._default_manager, "get_by_natural_key"):
325 keys = node.getElementsByTagName("natural")
326 if keys:
327 # If there are 'natural' subelements, it must be a natural key
328 field_value = [getInnerText(k).strip() for k in keys]
329 try:
330 obj = model._default_manager.db_manager(
331 self.db
332 ).get_by_natural_key(*field_value)
333 except ObjectDoesNotExist:
334 if self.handle_forward_references:
335 return base.DEFER_FIELD
336 else:
337 raise
338 obj_pk = getattr(obj, field.remote_field.field_name)
339 # If this is a natural foreign key to an object that
340 # has a FK/O2O as the foreign key, use the FK value
341 if field.remote_field.model._meta.pk.remote_field:
342 obj_pk = obj_pk.pk
343 else:
344 # Otherwise, treat like a normal PK
345 field_value = getInnerText(node).strip()
346 obj_pk = model._meta.get_field(
347 field.remote_field.field_name
348 ).to_python(field_value)
349 return obj_pk
350 else:
351 field_value = getInnerText(node).strip()
352 return model._meta.get_field(field.remote_field.field_name).to_python(
353 field_value
354 )
355
356 def _handle_m2m_field_node(self, node, field):
357 """
358 Handle a <field> node for a ManyToManyField.
359 """
360 model = field.remote_field.model
361 default_manager = model._default_manager
362 if hasattr(default_manager, "get_by_natural_key"):
363
364 def m2m_convert(n):
365 keys = n.getElementsByTagName("natural")
366 if keys:
367 # If there are 'natural' subelements, it must be a natural key
368 field_value = [getInnerText(k).strip() for k in keys]
369 obj_pk = (
370 default_manager.db_manager(self.db)
371 .get_by_natural_key(*field_value)
372 .pk
373 )
374 else:
375 # Otherwise, treat like a normal PK value.
376 obj_pk = model._meta.pk.to_python(n.getAttribute("pk"))
377 return obj_pk
378
379 else:
380
381 def m2m_convert(n):
382 return model._meta.pk.to_python(n.getAttribute("pk"))
383
384 values = []
385 try:
386 for c in node.getElementsByTagName("object"):
387 values.append(m2m_convert(c))
388 except Exception as e:
389 if isinstance(e, ObjectDoesNotExist) and self.handle_forward_references:
390 return base.DEFER_FIELD
391 else:
392 raise base.M2MDeserializationError(e, c)
393 else:
394 return values
395
396 def _get_model_from_node(self, node, attr):
397 """
398 Look up a model from a <object model=...> or a <field rel=... to=...>
399 node.
400 """
401 model_identifier = node.getAttribute(attr)
402 if not model_identifier:
403 raise base.DeserializationError(
404 "<%s> node is missing the required '%s' attribute"
405 % (node.nodeName, attr)
406 )
407 try:
408 return apps.get_model(model_identifier)
409 except (LookupError, TypeError):
410 raise base.DeserializationError(
411 "<%s> node has invalid model identifier: '%s'"
412 % (node.nodeName, model_identifier)
413 )
414
415
416 def getInnerText(node):
417 """Get all the inner text of a DOM node (recursively)."""
418 inner_text_list = getInnerTextList(node)
419 return "".join(inner_text_list)
420
421
422 def getInnerTextList(node):
423 """Return a list of the inner texts of a DOM node (recursively)."""
424 # inspired by https://mail.python.org/pipermail/xml-sig/2005-March/011022.html
425 result = []
426 for child in node.childNodes:
427 if (
428 child.nodeType == child.TEXT_NODE
429 or child.nodeType == child.CDATA_SECTION_NODE
430 ):
431 result.append(child.data)
432 elif child.nodeType == child.ELEMENT_NODE:
433 result.extend(getInnerTextList(child))
434 else:
435 pass
436 return result
437
438
439 # Below code based on Christian Heimes' defusedxml
440
441
442 class DefusedExpatParser(_ExpatParser):
443 """
444 An expat parser hardened against XML bomb attacks.
445
446 Forbid DTDs, external entity references
447 """
448
449 def __init__(self, *args, **kwargs):
450 super().__init__(*args, **kwargs)
451 self.setFeature(handler.feature_external_ges, False)
452 self.setFeature(handler.feature_external_pes, False)
453
454 def start_doctype_decl(self, name, sysid, pubid, has_internal_subset):
455 raise DTDForbidden(name, sysid, pubid)
456
457 def entity_decl(
458 self, name, is_parameter_entity, value, base, sysid, pubid, notation_name
459 ):
460 raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)
461
462 def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name):
463 # expat 1.2
464 raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name)
465
466 def external_entity_ref_handler(self, context, base, sysid, pubid):
467 raise ExternalReferenceForbidden(context, base, sysid, pubid)
468
469 def reset(self):
470 _ExpatParser.reset(self)
471 parser = self._parser
472 parser.StartDoctypeDeclHandler = self.start_doctype_decl
473 parser.EntityDeclHandler = self.entity_decl
474 parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl
475 parser.ExternalEntityRefHandler = self.external_entity_ref_handler
476
477
478 class DefusedXmlException(ValueError):
479 """Base exception."""
480
481 def __repr__(self):
482 return str(self)
483
484
485 class DTDForbidden(DefusedXmlException):
486 """Document type definition is forbidden."""
487
488 def __init__(self, name, sysid, pubid):
489 super().__init__()
490 self.name = name
491 self.sysid = sysid
492 self.pubid = pubid
493
494 def __str__(self):
495 tpl = "DTDForbidden(name='{}', system_id={!r}, public_id={!r})"
496 return tpl.format(self.name, self.sysid, self.pubid)
497
498
499 class EntitiesForbidden(DefusedXmlException):
500 """Entity definition is forbidden."""
501
502 def __init__(self, name, value, base, sysid, pubid, notation_name):
503 super().__init__()
504 self.name = name
505 self.value = value
506 self.base = base
507 self.sysid = sysid
508 self.pubid = pubid
509 self.notation_name = notation_name
510
511 def __str__(self):
512 tpl = "EntitiesForbidden(name='{}', system_id={!r}, public_id={!r})"
513 return tpl.format(self.name, self.sysid, self.pubid)
514
515
516 class ExternalReferenceForbidden(DefusedXmlException):
517 """Resolving an external reference is forbidden."""
518
519 def __init__(self, context, base, sysid, pubid):
520 super().__init__()
521 self.context = context
522 self.base = base
523 self.sysid = sysid
524 self.pubid = pubid
525
526 def __str__(self):
527 tpl = "ExternalReferenceForbidden(system_id='{}', public_id={})"
528 return tpl.format(self.sysid, self.pubid)