| 1 |
""" |
| 2 |
XML serializer. |
| 3 |
""" |
| 4 |
|
| 5 |
import json |
| 6 |
from contextlib import contextmanager |
| 7 |
from xml.dom import minidom, pulldom |
| 8 |
from xml.sax import handler |
| 9 |
from xml.sax.expatreader import ExpatParser as _ExpatParser |
| 10 |
|
| 11 |
from django.apps import apps |
| 12 |
from django.conf import settings |
| 13 |
from django.core.exceptions import ObjectDoesNotExist |
| 14 |
from django.core.serializers import base |
| 15 |
from django.db import DEFAULT_DB_ALIAS, models |
| 16 |
from django.utils.xmlutils import SimplerXMLGenerator, UnserializableContentError |
| 17 |
|
| 18 |
|
| 19 |
@contextmanager |
| 20 |
def fast_cache_clearing(): |
| 21 |
"""Workaround for performance issues in minidom document checks. |
| 22 |
|
| 23 |
Speeds up repeated DOM operations by skipping unnecessary full traversal |
| 24 |
of the DOM tree. |
| 25 |
""" |
| 26 |
module_helper_was_lambda = False |
| 27 |
if original_fn := getattr(minidom, "_in_document", None): |
| 28 |
module_helper_was_lambda = original_fn.__name__ == "<lambda>" |
| 29 |
if not module_helper_was_lambda: |
| 30 |
minidom._in_document = lambda node: bool(node.ownerDocument) |
| 31 |
try: |
| 32 |
yield |
| 33 |
finally: |
| 34 |
if original_fn and not module_helper_was_lambda: |
| 35 |
minidom._in_document = original_fn |
| 36 |
|
| 37 |
|
| 38 |
class Serializer(base.Serializer): |
| 39 |
"""Serialize a QuerySet to XML.""" |
| 40 |
|
| 41 |
def indent(self, level): |
| 42 |
if self.options.get("indent") is not None: |
| 43 |
self.xml.ignorableWhitespace( |
| 44 |
"\n" + " " * self.options.get("indent") * level |
| 45 |
) |
| 46 |
|
| 47 |
def start_serialization(self): |
| 48 |
""" |
| 49 |
Start serialization -- open the XML document and the root element. |
| 50 |
""" |
| 51 |
self.xml = SimplerXMLGenerator( |
| 52 |
self.stream, self.options.get("encoding", settings.DEFAULT_CHARSET) |
| 53 |
) |
| 54 |
self.xml.startDocument() |
| 55 |
self.xml.startElement("django-objects", {"version": "1.0"}) |
| 56 |
|
| 57 |
def end_serialization(self): |
| 58 |
""" |
| 59 |
End serialization -- end the document. |
| 60 |
""" |
| 61 |
self.indent(0) |
| 62 |
self.xml.endElement("django-objects") |
| 63 |
self.xml.endDocument() |
| 64 |
|
| 65 |
def start_object(self, obj): |
| 66 |
""" |
| 67 |
Called as each object is handled. |
| 68 |
""" |
| 69 |
if not hasattr(obj, "_meta"): |
| 70 |
raise base.SerializationError( |
| 71 |
"Non-model object (%s) encountered during serialization" % type(obj) |
| 72 |
) |
| 73 |
|
| 74 |
self.indent(1) |
| 75 |
attrs = {"model": str(obj._meta)} |
| 76 |
if not self.use_natural_primary_keys or not hasattr(obj, "natural_key"): |
| 77 |
obj_pk = obj.pk |
| 78 |
if obj_pk is not None: |
| 79 |
attrs["pk"] = obj._meta.pk.value_to_string(obj) |
| 80 |
|
| 81 |
self.xml.startElement("object", attrs) |
| 82 |
|
| 83 |
def end_object(self, obj): |
| 84 |
""" |
| 85 |
Called after handling all fields for an object. |
| 86 |
""" |
| 87 |
self.indent(1) |
| 88 |
self.xml.endElement("object") |
| 89 |
|
| 90 |
def handle_field(self, obj, field): |
| 91 |
""" |
| 92 |
Handle each field on an object (except for ForeignKeys and |
| 93 |
ManyToManyFields). |
| 94 |
""" |
| 95 |
self.indent(2) |
| 96 |
self.xml.startElement( |
| 97 |
"field", |
| 98 |
{ |
| 99 |
"name": field.name, |
| 100 |
"type": field.get_internal_type(), |
| 101 |
}, |
| 102 |
) |
| 103 |
|
| 104 |
# Get a "string version" of the object's data. |
| 105 |
if getattr(obj, field.name) is not None: |
| 106 |
value = field.value_to_string(obj) |
| 107 |
if field.get_internal_type() == "JSONField": |
| 108 |
# Dump value since JSONField.value_to_string() doesn't output |
| 109 |
# strings. |
| 110 |
value = json.dumps(value, cls=field.encoder) |
| 111 |
try: |
| 112 |
self.xml.characters(value) |
| 113 |
except UnserializableContentError: |
| 114 |
raise ValueError( |
| 115 |
"%s.%s (pk:%s) contains unserializable characters" |
| 116 |
% (obj.__class__.__name__, field.name, obj.pk) |
| 117 |
) |
| 118 |
else: |
| 119 |
self.xml.addQuickElement("None") |
| 120 |
|
| 121 |
self.xml.endElement("field") |
| 122 |
|
| 123 |
def handle_fk_field(self, obj, field): |
| 124 |
""" |
| 125 |
Handle a ForeignKey (they need to be treated slightly |
| 126 |
differently from regular fields). |
| 127 |
""" |
| 128 |
self._start_relational_field(field) |
| 129 |
related_att = getattr(obj, field.attname) |
| 130 |
if related_att is not None: |
| 131 |
if self.use_natural_foreign_keys and hasattr( |
| 132 |
field.remote_field.model, "natural_key" |
| 133 |
): |
| 134 |
related = getattr(obj, field.name) |
| 135 |
# If related object has a natural key, use it |
| 136 |
related = related.natural_key() |
| 137 |
# Iterable natural keys are rolled out as subelements |
| 138 |
for key_value in related: |
| 139 |
self.xml.startElement("natural", {}) |
| 140 |
self.xml.characters(str(key_value)) |
| 141 |
self.xml.endElement("natural") |
| 142 |
else: |
| 143 |
self.xml.characters(str(related_att)) |
| 144 |
else: |
| 145 |
self.xml.addQuickElement("None") |
| 146 |
self.xml.endElement("field") |
| 147 |
|
| 148 |
def handle_m2m_field(self, obj, field): |
| 149 |
""" |
| 150 |
Handle a ManyToManyField. Related objects are only serialized as |
| 151 |
references to the object's PK (i.e. the related *data* is not dumped, |
| 152 |
just the relation). |
| 153 |
""" |
| 154 |
if field.remote_field.through._meta.auto_created: |
| 155 |
self._start_relational_field(field) |
| 156 |
if self.use_natural_foreign_keys and hasattr( |
| 157 |
field.remote_field.model, "natural_key" |
| 158 |
): |
| 159 |
# If the objects in the m2m have a natural key, use it |
| 160 |
def handle_m2m(value): |
| 161 |
natural = value.natural_key() |
| 162 |
# Iterable natural keys are rolled out as subelements |
| 163 |
self.xml.startElement("object", {}) |
| 164 |
for key_value in natural: |
| 165 |
self.xml.startElement("natural", {}) |
| 166 |
self.xml.characters(str(key_value)) |
| 167 |
self.xml.endElement("natural") |
| 168 |
self.xml.endElement("object") |
| 169 |
|
| 170 |
def queryset_iterator(obj, field): |
| 171 |
attr = getattr(obj, field.name) |
| 172 |
chunk_size = ( |
| 173 |
2000 if getattr(attr, "prefetch_cache_name", None) else None |
| 174 |
) |
| 175 |
return attr.iterator(chunk_size) |
| 176 |
|
| 177 |
else: |
| 178 |
|
| 179 |
def handle_m2m(value): |
| 180 |
self.xml.addQuickElement("object", attrs={"pk": str(value.pk)}) |
| 181 |
|
| 182 |
def queryset_iterator(obj, field): |
| 183 |
query_set = getattr(obj, field.name).select_related(None).only("pk") |
| 184 |
chunk_size = 2000 if query_set._prefetch_related_lookups else None |
| 185 |
return query_set.iterator(chunk_size=chunk_size) |
| 186 |
|
| 187 |
m2m_iter = getattr(obj, "_prefetched_objects_cache", {}).get( |
| 188 |
field.name, |
| 189 |
queryset_iterator(obj, field), |
| 190 |
) |
| 191 |
for relobj in m2m_iter: |
| 192 |
handle_m2m(relobj) |
| 193 |
|
| 194 |
self.xml.endElement("field") |
| 195 |
|
| 196 |
def _start_relational_field(self, field): |
| 197 |
"""Output the <field> element for relational fields.""" |
| 198 |
self.indent(2) |
| 199 |
self.xml.startElement( |
| 200 |
"field", |
| 201 |
{ |
| 202 |
"name": field.name, |
| 203 |
"rel": field.remote_field.__class__.__name__, |
| 204 |
"to": str(field.remote_field.model._meta), |
| 205 |
}, |
| 206 |
) |
| 207 |
|
| 208 |
|
| 209 |
class Deserializer(base.Deserializer): |
| 210 |
"""Deserialize XML.""" |
| 211 |
|
| 212 |
def __init__( |
| 213 |
self, |
| 214 |
stream_or_string, |
| 215 |
*, |
| 216 |
using=DEFAULT_DB_ALIAS, |
| 217 |
ignorenonexistent=False, |
| 218 |
**options, |
| 219 |
): |
| 220 |
super().__init__(stream_or_string, **options) |
| 221 |
self.handle_forward_references = options.pop("handle_forward_references", False) |
| 222 |
self.event_stream = pulldom.parse(self.stream, self._make_parser()) |
| 223 |
self.db = using |
| 224 |
self.ignore = ignorenonexistent |
| 225 |
|
| 226 |
def _make_parser(self): |
| 227 |
"""Create a hardened XML parser (no custom/external entities).""" |
| 228 |
return DefusedExpatParser() |
| 229 |
|
| 230 |
def __next__(self): |
| 231 |
for event, node in self.event_stream: |
| 232 |
if event == "START_ELEMENT" and node.nodeName == "object": |
| 233 |
with fast_cache_clearing(): |
| 234 |
self.event_stream.expandNode(node) |
| 235 |
return self._handle_object(node) |
| 236 |
raise StopIteration |
| 237 |
|
| 238 |
def _handle_object(self, node): |
| 239 |
"""Convert an <object> node to a DeserializedObject.""" |
| 240 |
# Look up the model using the model loading mechanism. If this fails, |
| 241 |
# bail. |
| 242 |
Model = self._get_model_from_node(node, "model") |
| 243 |
|
| 244 |
# Start building a data dictionary from the object. |
| 245 |
data = {} |
| 246 |
if node.hasAttribute("pk"): |
| 247 |
data[Model._meta.pk.attname] = Model._meta.pk.to_python( |
| 248 |
node.getAttribute("pk") |
| 249 |
) |
| 250 |
|
| 251 |
# Also start building a dict of m2m data (this is saved as |
| 252 |
# {m2m_accessor_attribute : [list_of_related_objects]}) |
| 253 |
m2m_data = {} |
| 254 |
deferred_fields = {} |
| 255 |
|
| 256 |
field_names = {f.name for f in Model._meta.get_fields()} |
| 257 |
# Deserialize each field. |
| 258 |
for field_node in node.getElementsByTagName("field"): |
| 259 |
# If the field is missing the name attribute, bail (are you |
| 260 |
# sensing a pattern here?) |
| 261 |
field_name = field_node.getAttribute("name") |
| 262 |
if not field_name: |
| 263 |
raise base.DeserializationError( |
| 264 |
"<field> node is missing the 'name' attribute" |
| 265 |
) |
| 266 |
|
| 267 |
# Get the field from the Model. This will raise a |
| 268 |
# FieldDoesNotExist if, well, the field doesn't exist, which will |
| 269 |
# be propagated correctly unless ignorenonexistent=True is used. |
| 270 |
if self.ignore and field_name not in field_names: |
| 271 |
continue |
| 272 |
field = Model._meta.get_field(field_name) |
| 273 |
|
| 274 |
# As is usually the case, relation fields get the special treatment. |
| 275 |
if field.remote_field and isinstance( |
| 276 |
field.remote_field, models.ManyToManyRel |
| 277 |
): |
| 278 |
value = self._handle_m2m_field_node(field_node, field) |
| 279 |
if value == base.DEFER_FIELD: |
| 280 |
deferred_fields[field] = [ |
| 281 |
[ |
| 282 |
getInnerText(nat_node).strip() |
| 283 |
for nat_node in obj_node.getElementsByTagName("natural") |
| 284 |
] |
| 285 |
for obj_node in field_node.getElementsByTagName("object") |
| 286 |
] |
| 287 |
else: |
| 288 |
m2m_data[field.name] = value |
| 289 |
elif field.remote_field and isinstance( |
| 290 |
field.remote_field, models.ManyToOneRel |
| 291 |
): |
| 292 |
value = self._handle_fk_field_node(field_node, field) |
| 293 |
if value == base.DEFER_FIELD: |
| 294 |
deferred_fields[field] = [ |
| 295 |
getInnerText(k).strip() |
| 296 |
for k in field_node.getElementsByTagName("natural") |
| 297 |
] |
| 298 |
else: |
| 299 |
data[field.attname] = value |
| 300 |
else: |
| 301 |
if field_node.getElementsByTagName("None"): |
| 302 |
value = None |
| 303 |
else: |
| 304 |
value = field.to_python(getInnerText(field_node).strip()) |
| 305 |
# Load value since JSONField.to_python() outputs strings. |
| 306 |
if field.get_internal_type() == "JSONField": |
| 307 |
value = json.loads(value, cls=field.decoder) |
| 308 |
data[field.name] = value |
| 309 |
|
| 310 |
obj = base.build_instance(Model, data, self.db) |
| 311 |
|
| 312 |
# Return a DeserializedObject so that the m2m data has a place to live. |
| 313 |
return base.DeserializedObject(obj, m2m_data, deferred_fields) |
| 314 |
|
| 315 |
def _handle_fk_field_node(self, node, field): |
| 316 |
""" |
| 317 |
Handle a <field> node for a ForeignKey |
| 318 |
""" |
| 319 |
# Check if there is a child node named 'None', returning None if so. |
| 320 |
if node.getElementsByTagName("None"): |
| 321 |
return None |
| 322 |
else: |
| 323 |
model = field.remote_field.model |
| 324 |
if hasattr(model._default_manager, "get_by_natural_key"): |
| 325 |
keys = node.getElementsByTagName("natural") |
| 326 |
if keys: |
| 327 |
# If there are 'natural' subelements, it must be a natural key |
| 328 |
field_value = [getInnerText(k).strip() for k in keys] |
| 329 |
try: |
| 330 |
obj = model._default_manager.db_manager( |
| 331 |
self.db |
| 332 |
).get_by_natural_key(*field_value) |
| 333 |
except ObjectDoesNotExist: |
| 334 |
if self.handle_forward_references: |
| 335 |
return base.DEFER_FIELD |
| 336 |
else: |
| 337 |
raise |
| 338 |
obj_pk = getattr(obj, field.remote_field.field_name) |
| 339 |
# If this is a natural foreign key to an object that |
| 340 |
# has a FK/O2O as the foreign key, use the FK value |
| 341 |
if field.remote_field.model._meta.pk.remote_field: |
| 342 |
obj_pk = obj_pk.pk |
| 343 |
else: |
| 344 |
# Otherwise, treat like a normal PK |
| 345 |
field_value = getInnerText(node).strip() |
| 346 |
obj_pk = model._meta.get_field( |
| 347 |
field.remote_field.field_name |
| 348 |
).to_python(field_value) |
| 349 |
return obj_pk |
| 350 |
else: |
| 351 |
field_value = getInnerText(node).strip() |
| 352 |
return model._meta.get_field(field.remote_field.field_name).to_python( |
| 353 |
field_value |
| 354 |
) |
| 355 |
|
| 356 |
def _handle_m2m_field_node(self, node, field): |
| 357 |
""" |
| 358 |
Handle a <field> node for a ManyToManyField. |
| 359 |
""" |
| 360 |
model = field.remote_field.model |
| 361 |
default_manager = model._default_manager |
| 362 |
if hasattr(default_manager, "get_by_natural_key"): |
| 363 |
|
| 364 |
def m2m_convert(n): |
| 365 |
keys = n.getElementsByTagName("natural") |
| 366 |
if keys: |
| 367 |
# If there are 'natural' subelements, it must be a natural key |
| 368 |
field_value = [getInnerText(k).strip() for k in keys] |
| 369 |
obj_pk = ( |
| 370 |
default_manager.db_manager(self.db) |
| 371 |
.get_by_natural_key(*field_value) |
| 372 |
.pk |
| 373 |
) |
| 374 |
else: |
| 375 |
# Otherwise, treat like a normal PK value. |
| 376 |
obj_pk = model._meta.pk.to_python(n.getAttribute("pk")) |
| 377 |
return obj_pk |
| 378 |
|
| 379 |
else: |
| 380 |
|
| 381 |
def m2m_convert(n): |
| 382 |
return model._meta.pk.to_python(n.getAttribute("pk")) |
| 383 |
|
| 384 |
values = [] |
| 385 |
try: |
| 386 |
for c in node.getElementsByTagName("object"): |
| 387 |
values.append(m2m_convert(c)) |
| 388 |
except Exception as e: |
| 389 |
if isinstance(e, ObjectDoesNotExist) and self.handle_forward_references: |
| 390 |
return base.DEFER_FIELD |
| 391 |
else: |
| 392 |
raise base.M2MDeserializationError(e, c) |
| 393 |
else: |
| 394 |
return values |
| 395 |
|
| 396 |
def _get_model_from_node(self, node, attr): |
| 397 |
""" |
| 398 |
Look up a model from a <object model=...> or a <field rel=... to=...> |
| 399 |
node. |
| 400 |
""" |
| 401 |
model_identifier = node.getAttribute(attr) |
| 402 |
if not model_identifier: |
| 403 |
raise base.DeserializationError( |
| 404 |
"<%s> node is missing the required '%s' attribute" |
| 405 |
% (node.nodeName, attr) |
| 406 |
) |
| 407 |
try: |
| 408 |
return apps.get_model(model_identifier) |
| 409 |
except (LookupError, TypeError): |
| 410 |
raise base.DeserializationError( |
| 411 |
"<%s> node has invalid model identifier: '%s'" |
| 412 |
% (node.nodeName, model_identifier) |
| 413 |
) |
| 414 |
|
| 415 |
|
| 416 |
def getInnerText(node): |
| 417 |
"""Get all the inner text of a DOM node (recursively).""" |
| 418 |
inner_text_list = getInnerTextList(node) |
| 419 |
return "".join(inner_text_list) |
| 420 |
|
| 421 |
|
| 422 |
def getInnerTextList(node): |
| 423 |
"""Return a list of the inner texts of a DOM node (recursively).""" |
| 424 |
# inspired by https://mail.python.org/pipermail/xml-sig/2005-March/011022.html |
| 425 |
result = [] |
| 426 |
for child in node.childNodes: |
| 427 |
if ( |
| 428 |
child.nodeType == child.TEXT_NODE |
| 429 |
or child.nodeType == child.CDATA_SECTION_NODE |
| 430 |
): |
| 431 |
result.append(child.data) |
| 432 |
elif child.nodeType == child.ELEMENT_NODE: |
| 433 |
result.extend(getInnerTextList(child)) |
| 434 |
else: |
| 435 |
pass |
| 436 |
return result |
| 437 |
|
| 438 |
|
| 439 |
# Below code based on Christian Heimes' defusedxml |
| 440 |
|
| 441 |
|
| 442 |
class DefusedExpatParser(_ExpatParser): |
| 443 |
""" |
| 444 |
An expat parser hardened against XML bomb attacks. |
| 445 |
|
| 446 |
Forbid DTDs, external entity references |
| 447 |
""" |
| 448 |
|
| 449 |
def __init__(self, *args, **kwargs): |
| 450 |
super().__init__(*args, **kwargs) |
| 451 |
self.setFeature(handler.feature_external_ges, False) |
| 452 |
self.setFeature(handler.feature_external_pes, False) |
| 453 |
|
| 454 |
def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): |
| 455 |
raise DTDForbidden(name, sysid, pubid) |
| 456 |
|
| 457 |
def entity_decl( |
| 458 |
self, name, is_parameter_entity, value, base, sysid, pubid, notation_name |
| 459 |
): |
| 460 |
raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name) |
| 461 |
|
| 462 |
def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): |
| 463 |
# expat 1.2 |
| 464 |
raise EntitiesForbidden(name, None, base, sysid, pubid, notation_name) |
| 465 |
|
| 466 |
def external_entity_ref_handler(self, context, base, sysid, pubid): |
| 467 |
raise ExternalReferenceForbidden(context, base, sysid, pubid) |
| 468 |
|
| 469 |
def reset(self): |
| 470 |
_ExpatParser.reset(self) |
| 471 |
parser = self._parser |
| 472 |
parser.StartDoctypeDeclHandler = self.start_doctype_decl |
| 473 |
parser.EntityDeclHandler = self.entity_decl |
| 474 |
parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl |
| 475 |
parser.ExternalEntityRefHandler = self.external_entity_ref_handler |
| 476 |
|
| 477 |
|
| 478 |
class DefusedXmlException(ValueError): |
| 479 |
"""Base exception.""" |
| 480 |
|
| 481 |
def __repr__(self): |
| 482 |
return str(self) |
| 483 |
|
| 484 |
|
| 485 |
class DTDForbidden(DefusedXmlException): |
| 486 |
"""Document type definition is forbidden.""" |
| 487 |
|
| 488 |
def __init__(self, name, sysid, pubid): |
| 489 |
super().__init__() |
| 490 |
self.name = name |
| 491 |
self.sysid = sysid |
| 492 |
self.pubid = pubid |
| 493 |
|
| 494 |
def __str__(self): |
| 495 |
tpl = "DTDForbidden(name='{}', system_id={!r}, public_id={!r})" |
| 496 |
return tpl.format(self.name, self.sysid, self.pubid) |
| 497 |
|
| 498 |
|
| 499 |
class EntitiesForbidden(DefusedXmlException): |
| 500 |
"""Entity definition is forbidden.""" |
| 501 |
|
| 502 |
def __init__(self, name, value, base, sysid, pubid, notation_name): |
| 503 |
super().__init__() |
| 504 |
self.name = name |
| 505 |
self.value = value |
| 506 |
self.base = base |
| 507 |
self.sysid = sysid |
| 508 |
self.pubid = pubid |
| 509 |
self.notation_name = notation_name |
| 510 |
|
| 511 |
def __str__(self): |
| 512 |
tpl = "EntitiesForbidden(name='{}', system_id={!r}, public_id={!r})" |
| 513 |
return tpl.format(self.name, self.sysid, self.pubid) |
| 514 |
|
| 515 |
|
| 516 |
class ExternalReferenceForbidden(DefusedXmlException): |
| 517 |
"""Resolving an external reference is forbidden.""" |
| 518 |
|
| 519 |
def __init__(self, context, base, sysid, pubid): |
| 520 |
super().__init__() |
| 521 |
self.context = context |
| 522 |
self.base = base |
| 523 |
self.sysid = sysid |
| 524 |
self.pubid = pubid |
| 525 |
|
| 526 |
def __str__(self): |
| 527 |
tpl = "ExternalReferenceForbidden(system_id='{}', public_id={})" |
| 528 |
return tpl.format(self.sysid, self.pubid) |